Lua

Amazon S3 List More than 1000 Objects in Bucket

See more Amazon S3 Examples

S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens. Lua

local chilkat = require("chilkat")

local success = false

-- This example assumes the Chilkat HTTP API to have been previously unlocked.
-- See Global Unlock Sample for sample code.

local http = chilkat.Http.new()

-- Insert your access key here:
http.aws_access_key = "AWS_ACCESS_KEY"

-- Insert your secret key here:
http.aws_secret_key = "AWS_SECRET_KEY"

-- For the example, we'll get a listing containing approx. 25 objects
-- using continuation tokens with an artificially low max-keys set to 5.
-- (You may omit the max-keys query parameter to get the default 1000 maximum
-- number of keys per request.)

local sbContinuationToken = chilkat.StringBuilder.new()
local bGetMore = true
local sbUri = chilkat.StringBuilder.new()
local xml = chilkat.Xml.new()

local itemKey = ""
local itemSizeDecimalStr = ""
local lastModTimestamp = ""
local dt = chilkat.DateTime.new()
local dtObj = chilkat.DtObj.new()

while bGetMore do
    -- Make sure to set the "list-type" query param equal to "2".
    -- This chooses the GET Bucket (List Objects) Version 2 of the method.

    -- IMPORTANT: You must include the max-keys param to get a response with a continuation token.
    -- S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
    -- can be up to 1000.  If, for example, you have 2000 objects and do not specify
    -- max-keys, then the response will contain 1000 objects with no continuation token.
    sbUri:set_string("chilkat100?list-type=2&max-keys=5")
    if sbContinuationToken.length > 0 then
        sbUri:append("&continuation-token=")
        sbUri:append(sbContinuationToken:get_encoded("url", "utf-8"))
    end

    local strXml = http:s3_list_bucket_objects(sbUri:get_as_string())
    if not http.last_method_success then
        print(http.last_error_text)
        return
    end

    print("Response status code = " .. http.last_status)

    success = xml:load_xml(strXml)
    if not success then
        print(xml.last_error_text)
        return
    end

    -- If the response status code was not 200, then the XML response is not a
    -- listing of objects, but instead contains error information.
    if http.last_status ~= 200 then
        print(xml:get_xml())
        print("Failed.")
        return
    end

    -- If this is not the final response, then we'll get an XML response that begins
    -- like this.  (The IsTruncated child will be "true", and the NextContinuationToken
    -- will have a value to be used in the next request.)
    --     <?xml version="1.0" encoding="UTF-8" ?>
    --     <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    --         <Name>chilkat100</Name>
    --         <Prefix />
    --         <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
    --         <KeyCount>5</KeyCount>
    --         <MaxKeys>5</MaxKeys>
    --         <IsTruncated>true</IsTruncated>
    --

    -- Iterate over the bucket items in this chunk get information for each..
    local numItems = xml:num_children_having_tag("Contents")
    print("Number of bucket items = " .. numItems)

    local i = 0
    while i < numItems do
        xml.i = i

        itemKey = xml:get_child_content("Contents[i]|Key")
        itemSizeDecimalStr = xml:get_child_content("Contents[i]|Size")
        lastModTimestamp = xml:get_child_content("Contents[i]|LastModified")

        dt:set_from_rfc822(lastModTimestamp)
        -- Get a local date/time.
        local bLocal = true
        dt:to_dt_obj(bLocal, dtObj)

        print(i .. ": " .. itemKey .. ", " .. itemSizeDecimalStr .. ", " .. dtObj.day .. "-" .. dtObj.month .. "-" .. dtObj.year .. ":" .. dtObj.hour .. ":" .. dtObj.minute)

        i = i + 1
    end

    -- Check IsTruncated.
    bGetMore = false
    if xml:child_content_matches("IsTruncated", "true", true) then
        sbContinuationToken:set_string(xml:get_child_content("NextContinuationToken"))
        if sbContinuationToken.length > 0 then
            bGetMore = true
        end
    end
end

-- The output of this example (when tested) was:

-- Response status code = 200
-- Number of bucket items = 5
-- 0: Abc.ics, 1833, 25-5-2011:9:53
-- 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
-- 2: chiliPepper.gif, 7718, 12-3-2017:12:18
-- 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
-- 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
-- Response status code = 200
-- Number of bucket items = 5
-- 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
-- 1: dude.gif, 6373, 25-5-2011:17:29
-- 2: french.txt, 47, 12-3-2017:12:18
-- 3: hamlet.xml, 279658, 2-5-2016:12:21
-- 4: hamlet_play.xml, 279658, 20-3-2017:8:22
-- Response status code = 200
-- Number of bucket items = 5
-- 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
-- 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
-- 2: new folder/, 0, 26-11-2014:12:36
-- 3: new_starfish.jpg, 6229, 20-3-2017:8:22
-- 4: pigs.xml, 2804, 20-3-2017:8:22
-- Response status code = 200
-- Number of bucket items = 5
-- 0: somethingBig.zip, 13089458, 26-9-2016:9:29
-- 1: starfish.jpg, 6229, 12-3-2017:12:18
-- 2: starfish/, 0, 10-11-2014:10:7
-- 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
-- 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
-- Response status code = 200
-- Number of bucket items = 5
-- 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
-- 1: starfish2.jpg, 5987, 20-4-2012:12:6
-- 2: starfish3.jpg, 5987, 11-4-2012:7:10
-- 3: starfishA.jpg, 6229, 10-5-2016:8:44
-- 4: starfishCust.jpg, 6229, 12-11-2014:18:25
-- Response status code = 200
-- Number of bucket items = 1
-- 0: xyz.ics, 1833, 25-5-2011:8:52
--
--