Lua

S3 Upload the Parts for a Multipart Upload

See more Amazon S3 (new) Examples

This example uploads a large file in parts. The multipart upload needs to have been first initiated prior to uploading the parts.

See http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html for more information about uploading parts.

Lua

local chilkat = require("chilkat")

-- In the 1st step for uploading a large file, the multipart upload was initiated
-- as shown here: Initiate Multipart Upload

-- Other S3 Multipart Upload Examples:
-- Complete Multipart Upload
-- Abort Multipart Upload
-- List Parts

-- When we initiated the multipart upload, we saved the XML response to a file.  This
-- XML response contains the UploadId.  We'll begin by loading that XML and getting
-- the Upload ID.

local xmlInit = chilkat.Xml.new()
local success = xmlInit:load_xml_file("s3_multipart_uploads/initiate.xml")
if not success then
    print("Did not find the initiate.xml XML file.")
    return
end

local uploadId = xmlInit:get_child_content("UploadId")
print("UploadId = " .. uploadId)

-- When uploading parts, we need to keep an XML record of each part number
-- and its corresponding ETag, which is received in the response for each part.
-- There can be up to 10000 parts, numbered 1 to 10000.
-- After all parts have been uploaded, the final step will be to complete
-- the multipart upload (see Complete Multipart Upload)

-- In this example, the large file we want to upload is somethingBig.zip
local fileToUploadPath = "s3_multipart_uploads/somethingBig.zip"

-- The minimum allowed part size is 5MB (5242880 bytes).  The last part can be smaller because
-- it will contain the remainder of the file.  (This minimum is enforced by the AWS service.)
-- We'll use the minimum allowed part size for this example.
local partSize = 5242880

-- Let's use Chilkat's FileAccess API to examine the file to be uploaded.  We'll get the size
-- of the file and find out how many parts will be needed, including the final "partial" part.
local fac = chilkat.FileAccess.new()
fac:open_for_read(fileToUploadPath)

-- How many parts will there be if each part is 5242880 bytes?
local numParts = fac:get_num_blocks(partSize)
print("numParts = " .. numParts)
fac:file_close()

-- Imagine that we may be running this for the 1st time, or maybe we already
-- attempted to upload parts, and something failed. Maybe there was a network problem
-- the resulted in not all parts getting uploaded.  We'll write this code so that if run again,
-- it will upload whatever parts haven't yet been uploaded.

-- We'll keep a partsList.xml file to record the parts that have already been successfully
-- uploaded.  If this file does not yet exist, we'll create it..
local partsListFile = "s3_multipart_uploads/partsList.xml"
local partsListXml = chilkat.Xml.new()
if fac:file_exists(partsListFile) then
    partsListXml:load_xml_file(partsListFile)
end

-- Make sure the top-level tag is "CompleteMultipartUpload"
partsListXml.tag = "CompleteMultipartUpload"

-- --------------------------------------
-- Before entering the loop to upload parts,
-- setup the REST object with AWS authentication,
-- and make the initial connection.
local rest = chilkat.Rest.new()

-- Connect to the Amazon AWS REST server.
local bTls = true
local port = 443
local bAutoReconnect = true
success = rest:connect("s3.amazonaws.com", port, bTls, bAutoReconnect)

-- ----------------------------------------------------------------------------
-- Important: For buckets created in regions outside us-east-1,
-- there are three important changes that need to be made.
-- See Working with S3 Buckets in Non-us-east-1 Regions for the details.
-- ----------------------------------------------------------------------------

-- Provide AWS credentials for the REST call.
local authAws = chilkat.AuthAws.new()
authAws.access_key = "AWS_ACCESS_KEY"
authAws.secret_key = "AWS_SECRET_KEY"
authAws.service_name = "s3"
success = rest:set_auth_aws(authAws)

-- Set the bucket name via the HOST header.
-- In this case, the bucket name is "chilkat100".
rest.host = "chilkat100.s3.amazonaws.com"
-- --------------------------------------

local partNumber = 1
local sbPartNumber = chilkat.StringBuilder.new()

while partNumber <= numParts do
    print("---- " .. partNumber .. " ----")

    -- This cumbersome way of converting an integer to a string is because
    -- Chilkat examples are written in a script that is converted to many programming languages.
    -- At this time, the translator does not have integer-to-string code generation capability..
    sbPartNumber:clear()
    sbPartNumber:append_int(partNumber)

    local bPartAlreadyUploaded = false

    -- If there are no children, then the XML is empty and no parts have yet been uploaded.
    local numUploadedParts = partsListXml.num_children
    if numUploadedParts > 0 then
        -- If some parts have been uploaded, check to see if this particular part was already upload.
        -- If so, then it can be skipped.

        -- Position ourselves at the 1st record.
        local xRec0 = partsListXml:get_child(0)
        local foundRec = xRec0:find_next_record("PartNumber", sbPartNumber:get_as_string())
        if xRec0.last_method_success then
            bPartAlreadyUploaded = true
            print("Part " .. partNumber .. " was previously uploaded.")
            print(foundRec:get_xml())
        end
    end

    -- If this part was not already uploaded, we need to upload.
    -- Also update the partsListXml and save as each part is successfully uploaded.
    if not bPartAlreadyUploaded then
        print("Uploading part " .. partNumber .. " ...")

        -- Setup the stream source for the large file to be uploaded..
        local fileStream = chilkat.Stream.new()
        fileStream.source_file = fileToUploadPath
        -- The Chilkat Stream API has features to make uploading a parts
        -- of a file easy.  Indicate the part size by setting the SourceFilePartSize
        -- property.
        fileStream.source_file_part_size = partSize

        -- Our HTTP start line to upload a part will look like this:
        -- PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId HTTP/1.1

        -- Set the query params.  We'll need partNumber and uploadId.
        -- Make sure the query params from previous iterations are clear.
        rest:clear_all_query_params()
        rest:add_query_param("partNumber", sbPartNumber:get_as_string())
        rest:add_query_param("uploadId", uploadId)

        -- Upload this particular file part.
        -- Tell the fileStream which part is being uploaded.
        -- Our partNumber is 1-based (the 1st part is at index 1), but the fileStream's SourceFilePart
        -- property is 0-based.  Therefore we use partNumber-1.
        fileStream.source_file_part = partNumber - 1

        -- Because the SourceFilePart and SourceFilePartSize properties are set, the stream will
        -- will provide just that part of the file.
        local responseStr = rest:full_request_stream("PUT", "/somethingBig.zip", fileStream)
        if not rest.last_method_success then
            print(rest.last_error_text)
            return
        end

        if rest.response_status_code ~= 200 then
            -- Examine the request/response to see what happened.
            print("response status code = " .. rest.response_status_code)
            print("response status text = " .. rest.response_status_text)
            print("response header: " .. rest.response_header)
            print("response body: " .. responseStr)
            print("---")
            print("LastRequestStartLine: " .. rest.last_request_start_line)
            print("LastRequestHeader: " .. rest.last_request_header)
            return
        end

        -- OK, this part was uploaded..
        -- The response will have a 0-length body.  The only information we need is the
        -- ETag response header field.
        local etag = rest:response_hdr_by_name("ETag")
        -- It should be present, but just in case there was no ETag header...
        if not rest.last_method_success then
            print("No ETag response header found!")
            print("response header: " .. rest.response_header)
            return
        end

        -- We need to add record to the partsListXml.
        -- The record will look like this:
        -- &lt;Part>
        --   &lt;PartNumber>PartNumber&lt;/PartNumber>
        --   &lt;ETag>ETag&lt;/ETag>
        -- &lt;/Part>
        local xPart = partsListXml:new_child("Part", "")
        xPart:new_child_int2("PartNumber", partNumber)
        xPart:new_child2("ETag", etag)

        success = partsListXml:save_xml(partsListFile)
        if not success then
            print(partsListXml.last_error_text)
            return
        end

        print("-- Part " .. partNumber .. " uploaded. ---------------------")
    end

    partNumber = partNumber + 1
end

print("Finished.  All parts uploaded.")