C
C
Amazon S3 List More than 1000 Objects in Bucket
See more Amazon S3 Examples
S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens.Chilkat C Downloads
#include <C_CkHttp.h>
#include <C_CkStringBuilder.h>
#include <C_CkXml.h>
#include <C_CkDateTime.h>
#include <C_CkDtObj.h>
void ChilkatSample(void)
{
BOOL success;
HCkHttp http;
HCkStringBuilder sbContinuationToken;
BOOL bGetMore;
HCkStringBuilder sbUri;
HCkXml xml;
const char *itemKey;
const char *itemSizeDecimalStr;
const char *lastModTimestamp;
HCkDateTime dt;
HCkDtObj dtObj;
const char *strXml;
int numItems;
int i;
BOOL bLocal;
success = FALSE;
// This example assumes the Chilkat HTTP API to have been previously unlocked.
// See Global Unlock Sample for sample code.
http = CkHttp_Create();
// Insert your access key here:
CkHttp_putAwsAccessKey(http,"AWS_ACCESS_KEY");
// Insert your secret key here:
CkHttp_putAwsSecretKey(http,"AWS_SECRET_KEY");
// For the example, we'll get a listing containing approx. 25 objects
// using continuation tokens with an artificially low max-keys set to 5.
// (You may omit the max-keys query parameter to get the default 1000 maximum
// number of keys per request.)
sbContinuationToken = CkStringBuilder_Create();
bGetMore = TRUE;
sbUri = CkStringBuilder_Create();
xml = CkXml_Create();
dt = CkDateTime_Create();
dtObj = CkDtObj_Create();
while (bGetMore == TRUE) {
// Make sure to set the "list-type" query param equal to "2".
// This chooses the GET Bucket (List Objects) Version 2 of the method.
// IMPORTANT: You must include the max-keys param to get a response with a continuation token.
// S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
// can be up to 1000. If, for example, you have 2000 objects and do not specify
// max-keys, then the response will contain 1000 objects with no continuation token.
CkStringBuilder_SetString(sbUri,"chilkat100?list-type=2&max-keys=5");
if (CkStringBuilder_getLength(sbContinuationToken) > 0) {
CkStringBuilder_Append(sbUri,"&continuation-token=");
CkStringBuilder_Append(sbUri,CkStringBuilder_getEncoded(sbContinuationToken,"url","utf-8"));
}
strXml = CkHttp_s3_ListBucketObjects(http,CkStringBuilder_getAsString(sbUri));
if (CkHttp_getLastMethodSuccess(http) == FALSE) {
printf("%s\n",CkHttp_lastErrorText(http));
CkHttp_Dispose(http);
CkStringBuilder_Dispose(sbContinuationToken);
CkStringBuilder_Dispose(sbUri);
CkXml_Dispose(xml);
CkDateTime_Dispose(dt);
CkDtObj_Dispose(dtObj);
return;
}
printf("Response status code = %d\n",CkHttp_getLastStatus(http));
success = CkXml_LoadXml(xml,strXml);
if (success == FALSE) {
printf("%s\n",CkXml_lastErrorText(xml));
CkHttp_Dispose(http);
CkStringBuilder_Dispose(sbContinuationToken);
CkStringBuilder_Dispose(sbUri);
CkXml_Dispose(xml);
CkDateTime_Dispose(dt);
CkDtObj_Dispose(dtObj);
return;
}
// If the response status code was not 200, then the XML response is not a
// listing of objects, but instead contains error information.
if (CkHttp_getLastStatus(http) != 200) {
printf("%s\n",CkXml_getXml(xml));
printf("Failed.\n");
CkHttp_Dispose(http);
CkStringBuilder_Dispose(sbContinuationToken);
CkStringBuilder_Dispose(sbUri);
CkXml_Dispose(xml);
CkDateTime_Dispose(dt);
CkDtObj_Dispose(dtObj);
return;
}
// If this is not the final response, then we'll get an XML response that begins
// like this. (The IsTruncated child will be "true", and the NextContinuationToken
// will have a value to be used in the next request.)
// <?xml version="1.0" encoding="UTF-8" ?>
// <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
// <Name>chilkat100</Name>
// <Prefix />
// <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
// <KeyCount>5</KeyCount>
// <MaxKeys>5</MaxKeys>
// <IsTruncated>true</IsTruncated>
//
// Iterate over the bucket items in this chunk get information for each..
numItems = CkXml_NumChildrenHavingTag(xml,"Contents");
printf("Number of bucket items = %d\n",numItems);
i = 0;
while (i < numItems) {
CkXml_putI(xml,i);
itemKey = CkXml_getChildContent(xml,"Contents[i]|Key");
itemSizeDecimalStr = CkXml_getChildContent(xml,"Contents[i]|Size");
lastModTimestamp = CkXml_getChildContent(xml,"Contents[i]|LastModified");
CkDateTime_SetFromRfc822(dt,lastModTimestamp);
// Get a local date/time.
bLocal = TRUE;
CkDateTime_ToDtObj(dt,bLocal,dtObj);
printf("%d: %s, %s, %d-%d-%d:%d:%d\n",i,itemKey,itemSizeDecimalStr,CkDtObj_getDay(dtObj),CkDtObj_getMonth(dtObj),CkDtObj_getYear(dtObj)
,CkDtObj_getHour(dtObj),CkDtObj_getMinute(dtObj));
i = i + 1;
}
// Check IsTruncated.
bGetMore = FALSE;
if (CkXml_ChildContentMatches(xml,"IsTruncated","true",TRUE) == TRUE) {
CkStringBuilder_SetString(sbContinuationToken,CkXml_getChildContent(xml,"NextContinuationToken"));
if (CkStringBuilder_getLength(sbContinuationToken) > 0) {
bGetMore = TRUE;
}
}
}
// The output of this example (when tested) was:
// Response status code = 200
// Number of bucket items = 5
// 0: Abc.ics, 1833, 25-5-2011:9:53
// 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
// 2: chiliPepper.gif, 7718, 12-3-2017:12:18
// 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
// 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
// Response status code = 200
// Number of bucket items = 5
// 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
// 1: dude.gif, 6373, 25-5-2011:17:29
// 2: french.txt, 47, 12-3-2017:12:18
// 3: hamlet.xml, 279658, 2-5-2016:12:21
// 4: hamlet_play.xml, 279658, 20-3-2017:8:22
// Response status code = 200
// Number of bucket items = 5
// 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
// 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
// 2: new folder/, 0, 26-11-2014:12:36
// 3: new_starfish.jpg, 6229, 20-3-2017:8:22
// 4: pigs.xml, 2804, 20-3-2017:8:22
// Response status code = 200
// Number of bucket items = 5
// 0: somethingBig.zip, 13089458, 26-9-2016:9:29
// 1: starfish.jpg, 6229, 12-3-2017:12:18
// 2: starfish/, 0, 10-11-2014:10:7
// 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
// 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
// Response status code = 200
// Number of bucket items = 5
// 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
// 1: starfish2.jpg, 5987, 20-4-2012:12:6
// 2: starfish3.jpg, 5987, 11-4-2012:7:10
// 3: starfishA.jpg, 6229, 10-5-2016:8:44
// 4: starfishCust.jpg, 6229, 12-11-2014:18:25
// Response status code = 200
// Number of bucket items = 1
// 0: xyz.ics, 1833, 25-5-2011:8:52
//
//
CkHttp_Dispose(http);
CkStringBuilder_Dispose(sbContinuationToken);
CkStringBuilder_Dispose(sbUri);
CkXml_Dispose(xml);
CkDateTime_Dispose(dt);
CkDtObj_Dispose(dtObj);
}