PowerBuilder

Amazon S3 List More than 1000 Objects in Bucket

See more Amazon S3 Examples

S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens. PowerBuilder

integer li_rc
integer li_Success
oleobject loo_Http
oleobject loo_SbContinuationToken
integer li_BGetMore
oleobject loo_SbUri
oleobject loo_Xml
string ls_ItemKey
string ls_ItemSizeDecimalStr
string ls_LastModTimestamp
oleobject loo_Dt
oleobject loo_DtObj
string ls_StrXml
integer li_NumItems
integer i
integer li_BLocal

li_Success = 0

// This example assumes the Chilkat HTTP API to have been previously unlocked.
// See Global Unlock Sample for sample code.

loo_Http = create oleobject
li_rc = loo_Http.ConnectToNewObject("Chilkat.Http")
if li_rc < 0 then
    destroy loo_Http
    MessageBox("Error","Connecting to COM object failed")
    return
end if

// Insert your access key here:
loo_Http.AwsAccessKey = "AWS_ACCESS_KEY"

// Insert your secret key here:
loo_Http.AwsSecretKey = "AWS_SECRET_KEY"

// For the example, we'll get a listing containing approx. 25 objects
// using continuation tokens with an artificially low max-keys set to 5.
// (You may omit the max-keys query parameter to get the default 1000 maximum
// number of keys per request.)

loo_SbContinuationToken = create oleobject
li_rc = loo_SbContinuationToken.ConnectToNewObject("Chilkat.StringBuilder")

li_BGetMore = 1
loo_SbUri = create oleobject
li_rc = loo_SbUri.ConnectToNewObject("Chilkat.StringBuilder")

loo_Xml = create oleobject
li_rc = loo_Xml.ConnectToNewObject("Chilkat.Xml")

loo_Dt = create oleobject
li_rc = loo_Dt.ConnectToNewObject("Chilkat.CkDateTime")

loo_DtObj = create oleobject
li_rc = loo_DtObj.ConnectToNewObject("Chilkat.DtObj")

do while li_BGetMore = 1

    // Make sure to set the "list-type" query param equal to "2".
    // This chooses the GET Bucket (List Objects) Version 2 of the method.

    // IMPORTANT: You must include the max-keys param to get a response with a continuation token.
    // S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
    // can be up to 1000.  If, for example, you have 2000 objects and do not specify
    // max-keys, then the response will contain 1000 objects with no continuation token. 
    loo_SbUri.SetString("chilkat100?list-type=2&max-keys=5")
    if loo_SbContinuationToken.Length > 0 then
        loo_SbUri.Append("&continuation-token=")
        loo_SbUri.Append(loo_SbContinuationToken.GetEncoded("url","utf-8"))
    end if

    ls_StrXml = loo_Http.S3_ListBucketObjects(loo_SbUri.GetAsString())
    if loo_Http.LastMethodSuccess = 0 then
        Write-Debug loo_Http.LastErrorText
        destroy loo_Http
        destroy loo_SbContinuationToken
        destroy loo_SbUri
        destroy loo_Xml
        destroy loo_Dt
        destroy loo_DtObj
        return
    end if

    Write-Debug "Response status code = " + string(loo_Http.LastStatus)

    li_Success = loo_Xml.LoadXml(ls_StrXml)
    if li_Success = 0 then
        Write-Debug loo_Xml.LastErrorText
        destroy loo_Http
        destroy loo_SbContinuationToken
        destroy loo_SbUri
        destroy loo_Xml
        destroy loo_Dt
        destroy loo_DtObj
        return
    end if

    // If the response status code was not 200, then the XML response is not a 
    // listing of objects, but instead contains error information.
    if loo_Http.LastStatus <> 200 then
        Write-Debug loo_Xml.GetXml()
        Write-Debug "Failed."
        destroy loo_Http
        destroy loo_SbContinuationToken
        destroy loo_SbUri
        destroy loo_Xml
        destroy loo_Dt
        destroy loo_DtObj
        return
    end if

    // If this is not the final response, then we'll get an XML response that begins
    // like this.  (The IsTruncated child will be "true", and the NextContinuationToken
    // will have a value to be used in the next request.)
    // 	<?xml version="1.0" encoding="UTF-8" ?>
    // 	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    // 	    <Name>chilkat100</Name>
    // 	    <Prefix />
    // 	    <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
    // 	    <KeyCount>5</KeyCount>
    // 	    <MaxKeys>5</MaxKeys>
    // 	    <IsTruncated>true</IsTruncated>
    // 	

    // Iterate over the bucket items in this chunk get information for each..
    li_NumItems = loo_Xml.NumChildrenHavingTag("Contents")
    Write-Debug "Number of bucket items = " + string(li_NumItems)

    i = 0
    do while i < li_NumItems
        loo_Xml.I = i

        ls_ItemKey = loo_Xml.GetChildContent("Contents[i]|Key")
        ls_ItemSizeDecimalStr = loo_Xml.GetChildContent("Contents[i]|Size")
        ls_LastModTimestamp = loo_Xml.GetChildContent("Contents[i]|LastModified")

        loo_Dt.SetFromRfc822(ls_LastModTimestamp)
        // Get a local date/time.
        li_BLocal = 1
        loo_Dt.ToDtObj(li_BLocal,loo_DtObj)

        Write-Debug string(i) + ": " + ls_ItemKey + ", " + ls_ItemSizeDecimalStr + ", " + string(loo_DtObj.Day) + "-" + string(loo_DtObj.Month) + "-" + string(loo_DtObj.Year) + ":" + string(loo_DtObj.Hour) + ":" + string(loo_DtObj.Minute)

        i = i + 1
    loop

    // Check IsTruncated.
    li_BGetMore = 0
    if loo_Xml.ChildContentMatches("IsTruncated","true",1) = 1 then
        loo_SbContinuationToken.SetString(loo_Xml.GetChildContent("NextContinuationToken"))
        if loo_SbContinuationToken.Length > 0 then
            li_BGetMore = 1
        end if

    end if

loop

// The output of this example (when tested) was:

// Response status code = 200
// Number of bucket items = 5
// 0: Abc.ics, 1833, 25-5-2011:9:53
// 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
// 2: chiliPepper.gif, 7718, 12-3-2017:12:18
// 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
// 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
// Response status code = 200
// Number of bucket items = 5
// 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
// 1: dude.gif, 6373, 25-5-2011:17:29
// 2: french.txt, 47, 12-3-2017:12:18
// 3: hamlet.xml, 279658, 2-5-2016:12:21
// 4: hamlet_play.xml, 279658, 20-3-2017:8:22
// Response status code = 200
// Number of bucket items = 5
// 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
// 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
// 2: new folder/, 0, 26-11-2014:12:36
// 3: new_starfish.jpg, 6229, 20-3-2017:8:22
// 4: pigs.xml, 2804, 20-3-2017:8:22
// Response status code = 200
// Number of bucket items = 5
// 0: somethingBig.zip, 13089458, 26-9-2016:9:29
// 1: starfish.jpg, 6229, 12-3-2017:12:18
// 2: starfish/, 0, 10-11-2014:10:7
// 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
// 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
// Response status code = 200
// Number of bucket items = 5
// 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
// 1: starfish2.jpg, 5987, 20-4-2012:12:6
// 2: starfish3.jpg, 5987, 11-4-2012:7:10
// 3: starfishA.jpg, 6229, 10-5-2016:8:44
// 4: starfishCust.jpg, 6229, 12-11-2014:18:25
// Response status code = 200
// Number of bucket items = 1
// 0: xyz.ics, 1833, 25-5-2011:8:52
// 
// 


destroy loo_Http
destroy loo_SbContinuationToken
destroy loo_SbUri
destroy loo_Xml
destroy loo_Dt
destroy loo_DtObj