View Javadoc
1   package com.github.davidmoten.aws.lw.client.internal.auth;
2   
3   import java.io.ByteArrayInputStream;
4   import java.io.DataOutputStream;
5   import java.net.HttpURLConnection;
6   import java.net.URL;
7   import java.util.HashMap;
8   import java.util.Map;
9   
10  import com.github.davidmoten.aws.lw.client.internal.util.Util;
11  
12  /**
13   * Sample code showing how to PUT objects to Amazon S3 using chunked uploading
14   * with Signature V4 authorization
15   */
16  public class PutS3ObjectChunkedSample {
17      
18      private static final String contentSeed = 
19              "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc tortor metus, sagittis eget augue ut,\n"
20              + "feugiat vehicula risus. Integer tortor mauris, vehicula nec mollis et, consectetur eget tortor. In ut\n"
21              + "elit sagittis, ultrices est ut, iaculis turpis. In hac habitasse platea dictumst. Donec laoreet tellus\n"
22              + "at auctor tempus. Praesent nec diam sed urna sollicitudin vehicula eget id est. Vivamus sed laoreet\n"
23              + "lectus. Aliquam convallis condimentum risus, vitae porta justo venenatis vitae. Phasellus vitae nunc\n"
24              + "varius, volutpat quam nec, mollis urna. Donec tempus, nisi vitae gravida facilisis, sapien sem malesuada\n"
25              + "purus, id semper libero ipsum condimentum nulla. Suspendisse vel mi leo. Morbi pellentesque placerat congue.\n"
26              + "Nunc sollicitudin nunc diam, nec hendrerit dui commodo sed. Duis dapibus commodo elit, id commodo erat\n"
27              + "congue id. Aliquam erat volutpat.\n";
28      
29      /**
30       * Uploads content to an Amazon S3 object in a series of signed 'chunks' using Signature V4 authorization.
31       */
32      public static void putS3ObjectChunked(String bucketName, String regionName, String awsAccessKey, String awsSecretKey) {
33          System.out.println("***************************************************");
34          System.out.println("*      Executing sample 'PutS3ObjectChunked'      *");
35          System.out.println("***************************************************");
36          
37          // this sample uses a chunk data length of 64K; this should yield one
38          // 64K chunk, one partial chunk and the final 0 byte payload terminator chunk
39          final int userDataBlockSize = 64 * 1024;
40          String sampleContent = make65KPayload();
41          
42          URL endpointUrl;
43              if (regionName.equals("us-east-1")) {
44                  endpointUrl = Util.toUrl("https://s3.amazonaws.com/" + bucketName + "/ExampleChunkedObject.txt");
45              } else {
46                  endpointUrl = Util.toUrl("https://s3-" + regionName + ".amazonaws.com/" + bucketName + "/ExampleChunkedObject.txt");
47              }
48          
49          // set the markers indicating we're going to send the upload as a series 
50          // of chunks:
51          //   -- 'x-amz-content-sha256' is the fixed marker indicating chunked
52          //      upload
53          //   -- 'content-length' becomes the total size in bytes of the upload 
54          //      (including chunk headers), 
55          //   -- 'x-amz-decoded-content-length' is used to transmit the actual 
56          //      length of the data payload, less chunk headers
57          
58          Map<String, String> headers = new HashMap<String, String>();
59          headers.put("x-amz-storage-class", "REDUCED_REDUNDANCY");
60          headers.put("x-amz-content-sha256", Aws4SignerForChunkedUpload.STREAMING_BODY_SHA256);
61          headers.put("content-encoding", "" + "aws-chunked");
62          headers.put("x-amz-decoded-content-length", "" + sampleContent.length());
63          
64          Aws4SignerForChunkedUpload signer = new Aws4SignerForChunkedUpload(
65                  endpointUrl, "PUT", "s3", regionName);
66          
67          // how big is the overall request stream going to be once we add the signature 
68          // 'headers' to each chunk?
69          long totalLength = Aws4SignerForChunkedUpload.calculateChunkedContentLength(sampleContent.length(), userDataBlockSize);
70          headers.put("content-length", "" + totalLength);
71          
72          String authorization = signer.computeSignature(headers, 
73                                                         null, // no query parameters
74                                                         Aws4SignerForChunkedUpload.STREAMING_BODY_SHA256, 
75                                                         awsAccessKey, 
76                                                         awsSecretKey);
77                  
78          // place the computed signature into a formatted 'Authorization' header 
79          // and call S3
80          headers.put("Authorization", authorization);
81          
82          // start consuming the data payload in blocks which we subsequently chunk; this prefixes
83          // the data with a 'chunk header' containing signature data from the prior chunk (or header
84          // signing, if the first chunk) plus length and other data. Each completed chunk is
85          // written to the request stream and to complete the upload, we send a final chunk with
86          // a zero-length data payload.
87          
88          try {
89              // first set up the connection
90              HttpURLConnection connection = HttpUtils.createHttpConnection(endpointUrl, "PUT", headers);
91              
92              // get the request stream and start writing the user data as chunks, as outlined
93              // above;
94              byte[] buffer = new byte[userDataBlockSize];
95              DataOutputStream outputStream = new DataOutputStream(connection.getOutputStream());
96              
97              // get the data stream
98              ByteArrayInputStream inputStream = new ByteArrayInputStream(sampleContent.getBytes("UTF-8"));
99              
100             int bytesRead = 0;
101             while ( (bytesRead = inputStream.read(buffer, 0, buffer.length)) != -1 ) {
102                 // process into a chunk
103                 byte[] chunk = signer.constructSignedChunk(bytesRead, buffer);
104                 
105                 // send the chunk
106                 outputStream.write(chunk);
107                 outputStream.flush();
108             }
109             
110             // last step is to send a signed zero-length chunk to complete the upload
111             byte[] finalChunk = signer.constructSignedChunk(0, buffer);
112             outputStream.write(finalChunk);
113             outputStream.flush();
114             outputStream.close();
115             
116             // make the call to Amazon S3
117             String response = HttpUtils.executeHttpRequest(connection);
118             System.out.println("--------- Response content ---------");
119             System.out.println(response);
120             System.out.println("------------------------------------");
121         } catch (Exception e) {
122             throw new RuntimeException("Error when sending chunked upload request. " + e.getMessage(), e);
123         }
124     }
125 
126     /**
127      * Want sample to upload 3 chunks for our selected chunk size of 64K; one
128      * full size chunk, one partial chunk and then the 0-byte terminator chunk.
129      * This routine just takes 1K of seed text and turns it into a 65K-or-so
130      * string for sample use.
131      */
132     private static String make65KPayload() {
133         StringBuilder oneKSeed = new StringBuilder();
134         while ( oneKSeed.length() < 1024 ) {
135             oneKSeed.append(contentSeed);
136         }
137         
138         // now scale up to meet/exceed our requirement
139         StringBuilder output = new StringBuilder();
140         for (int i = 0; i < 66; i++) {
141             output.append(oneKSeed);
142         }
143         return output.toString();
144     }
145 }