Tcl
Amazon S3 List More than 1000 Objects in Bucket

S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens.
Chilkat Tcl Downloads

Download Chilkat for Tcl
Tcl

load ./chilkat.dll

set success 0

# This example assumes the Chilkat HTTP API to have been previously unlocked.
# See Global Unlock Sample for sample code.

set http [new_CkHttp]

# Insert your access key here:
CkHttp_put_AwsAccessKey $http "AWS_ACCESS_KEY"

# Insert your secret key here:
CkHttp_put_AwsSecretKey $http "AWS_SECRET_KEY"

# For the example, we'll get a listing containing approx. 25 objects
# using continuation tokens with an artificially low max-keys set to 5.
# (You may omit the max-keys query parameter to get the default 1000 maximum
# number of keys per request.)

set sbContinuationToken [new_CkStringBuilder]

set bGetMore 1
set sbUri [new_CkStringBuilder]

set xml [new_CkXml]

set dt [new_CkDateTime]

set dtObj [new_CkDtObj]

while {$bGetMore == 1} {

    # Make sure to set the "list-type" query param equal to "2".
    # This chooses the GET Bucket (List Objects) Version 2 of the method.

    # IMPORTANT: You must include the max-keys param to get a response with a continuation token.
    # S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
    # can be up to 1000.  If, for example, you have 2000 objects and do not specify
    # max-keys, then the response will contain 1000 objects with no continuation token. 
    CkStringBuilder_SetString $sbUri "chilkat100?list-type=2&max-keys=5"
    if {[CkStringBuilder_get_Length $sbContinuationToken] > 0} then {
        CkStringBuilder_Append $sbUri "&continuation-token="
        CkStringBuilder_Append $sbUri [CkStringBuilder_getEncoded $sbContinuationToken "url" "utf-8"]
    }

    set strXml [CkHttp_s3_ListBucketObjects $http [CkStringBuilder_getAsString $sbUri]]
    if {[CkHttp_get_LastMethodSuccess $http] == 0} then {
        puts [CkHttp_lastErrorText $http]
        delete_CkHttp $http
        delete_CkStringBuilder $sbContinuationToken
        delete_CkStringBuilder $sbUri
        delete_CkXml $xml
        delete_CkDateTime $dt
        delete_CkDtObj $dtObj
        exit
    }

    puts "Response status code = [CkHttp_get_LastStatus $http]"

    set success [CkXml_LoadXml $xml $strXml]
    if {$success == 0} then {
        puts [CkXml_lastErrorText $xml]
        delete_CkHttp $http
        delete_CkStringBuilder $sbContinuationToken
        delete_CkStringBuilder $sbUri
        delete_CkXml $xml
        delete_CkDateTime $dt
        delete_CkDtObj $dtObj
        exit
    }

    # If the response status code was not 200, then the XML response is not a 
    # listing of objects, but instead contains error information.
    if {[CkHttp_get_LastStatus $http] != 200} then {
        puts [CkXml_getXml $xml]
        puts "Failed."
        delete_CkHttp $http
        delete_CkStringBuilder $sbContinuationToken
        delete_CkStringBuilder $sbUri
        delete_CkXml $xml
        delete_CkDateTime $dt
        delete_CkDtObj $dtObj
        exit
    }

    # If this is not the final response, then we'll get an XML response that begins
    # like this.  (The IsTruncated child will be "true", and the NextContinuationToken
    # will have a value to be used in the next request.)
    # 	<?xml version="1.0" encoding="UTF-8" ?>
    # 	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    # 	    <Name>chilkat100</Name>
    # 	    <Prefix />
    # 	    <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
    # 	    <KeyCount>5</KeyCount>
    # 	    <MaxKeys>5</MaxKeys>
    # 	    <IsTruncated>true</IsTruncated>
    # 	

    # Iterate over the bucket items in this chunk get information for each..
    set numItems [CkXml_NumChildrenHavingTag $xml "Contents"]
    puts "Number of bucket items = $numItems"

    set i 0
    while {$i < $numItems} {
        CkXml_put_I $xml $i

        set itemKey [CkXml_getChildContent $xml "Contents[i]|Key"]
        set itemSizeDecimalStr [CkXml_getChildContent $xml "Contents[i]|Size"]
        set lastModTimestamp [CkXml_getChildContent $xml "Contents[i]|LastModified"]

        CkDateTime_SetFromRfc822 $dt $lastModTimestamp
        # Get a local date/time.
        set bLocal 1
        CkDateTime_ToDtObj $dt $bLocal $dtObj

        puts "$i: $itemKey, $itemSizeDecimalStr, [CkDtObj_get_Day $dtObj]-[CkDtObj_get_Month $dtObj]-[CkDtObj_get_Year $dtObj]:[CkDtObj_get_Hour $dtObj]:[CkDtObj_get_Minute $dtObj]"

        set i [expr $i + 1]
    }

    # Check IsTruncated.
    set bGetMore 0
    if {[CkXml_ChildContentMatches $xml "IsTruncated" "true" 1] == 1} then {
        CkStringBuilder_SetString $sbContinuationToken [CkXml_getChildContent $xml "NextContinuationToken"]
        if {[CkStringBuilder_get_Length $sbContinuationToken] > 0} then {
            set bGetMore 1
        }

    }

}

# The output of this example (when tested) was:

# Response status code = 200
# Number of bucket items = 5
# 0: Abc.ics, 1833, 25-5-2011:9:53
# 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
# 2: chiliPepper.gif, 7718, 12-3-2017:12:18
# 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
# 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
# Response status code = 200
# Number of bucket items = 5
# 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
# 1: dude.gif, 6373, 25-5-2011:17:29
# 2: french.txt, 47, 12-3-2017:12:18
# 3: hamlet.xml, 279658, 2-5-2016:12:21
# 4: hamlet_play.xml, 279658, 20-3-2017:8:22
# Response status code = 200
# Number of bucket items = 5
# 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
# 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
# 2: new folder/, 0, 26-11-2014:12:36
# 3: new_starfish.jpg, 6229, 20-3-2017:8:22
# 4: pigs.xml, 2804, 20-3-2017:8:22
# Response status code = 200
# Number of bucket items = 5
# 0: somethingBig.zip, 13089458, 26-9-2016:9:29
# 1: starfish.jpg, 6229, 12-3-2017:12:18
# 2: starfish/, 0, 10-11-2014:10:7
# 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
# 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
# Response status code = 200
# Number of bucket items = 5
# 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
# 1: starfish2.jpg, 5987, 20-4-2012:12:6
# 2: starfish3.jpg, 5987, 11-4-2012:7:10
# 3: starfishA.jpg, 6229, 10-5-2016:8:44
# 4: starfishCust.jpg, 6229, 12-11-2014:18:25
# Response status code = 200
# Number of bucket items = 1
# 0: xyz.ics, 1833, 25-5-2011:8:52
# 
# 

delete_CkHttp $http
delete_CkStringBuilder $sbContinuationToken
delete_CkStringBuilder $sbUri
delete_CkXml $xml
delete_CkDateTime $dt
delete_CkDtObj $dtObj