(PureBasic) Google Drive - Build a Local Cache of Metadata

This example demonstrates how to download the metadata for all files in a Google Drive account to create a local filesystem cache with the information. The cache can be used to fetch information without having to query Google Drive.

Note: This example requires Chilkat v11.0.0 or greater.

Chilkat PureBasic Module Download Chilkat PureBasic Module

IncludeFile "CkJsonObject.pb"
IncludeFile "CkDateTime.pb"
IncludeFile "CkJsonArray.pb"
IncludeFile "CkCache.pb"
IncludeFile "CkHashtable.pb"
IncludeFile "CkStringBuilder.pb"
IncludeFile "CkAuthGoogle.pb"
IncludeFile "CkRest.pb"
IncludeFile "CkStringArray.pb"

Procedure ChilkatExample()

    success.i = 0

    success = 1

    ; It requires the Chilkat API to have been previously unlocked.
    ; See Global Unlock Sample for sample code.

    ; This example uses a previously obtained access token having permission for the 
    ; Google Drive scope.

    gAuth.i = CkAuthGoogle::ckCreate()
    If gAuth.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkAuthGoogle::setCkAccessToken(gAuth, "GOOGLE_DRIVE_ACCESS_TOKEN")

    rest.i = CkRest::ckCreate()
    If rest.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    ; Connect using TLS.
    bAutoReconnect.i = 1
    success = CkRest::ckConnect(rest,"www.googleapis.com",443,1,bAutoReconnect)

    ; Provide the authentication credentials (i.e. the access token)
    CkRest::ckSetAuthGoogle(rest,gAuth)

    ; -------------------------------------------------------------------
    ; Initialize our cache object.  Indicate the location of the root cache directory, and how many cache levels are to exist.
    ; For small caches (level 0) all cache files are in the root directory.
    ; For medium caches (level 1) cache files are located in 256 sub-directories from the root.
    ; For large caches (level 2) cache files are located in 256x256 sub-directories two levels down from the root.
    gdCache.i = CkCache::ckCreate()
    If gdCache.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkCache::setCkLevel(gdCache, 0)
    ; Use a root directory that makes sense on your operating system..
    CkCache::ckAddRoot(gdCache,"C:/ckCache/googleDrive")

    ; If we are re-building the cache, we can first delete the entire contents of the cache.
    numCacheFilesDeleted.i = CkCache::ckDeleteAll(gdCache)

    ; Create a date/time object with an time 7 days from the current date/time.
    dtExpire.i = CkDateTime::ckCreate()
    If dtExpire.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkDateTime::ckSetFromCurrentSystemTime(dtExpire)
    CkDateTime::ckAddDays(dtExpire,7)

    ; Indicate that we want ALL possible fields.
    ; If no fields are indicated, then only the basic fields are returned.
    allFields.s = "appProperties,capabilities,contentHints,createdTime,description,explicitlyTrashed,fileExtension,folderColorRgb,fullFileExtension,headRevisionId,iconLink,id,imageMediaMetadata,isAppAuthorized,kind,lastModifyingUser,md5Checksum,mimeType,modifiedByMeTime,modifiedTime,name,originalFilename,ownedByMe,owners,parents,permissions,properties,quotaBytesUsed,shared,sharedWithMeTime,sharingUser,size,spaces,starred,thumbnailLink,trashed,version,videoMediaMetadata,viewedByMe,viewedByMeTime,viewersCanCopyContent,webContentLink,webViewLink,writersCanShare"

    ; We're going to keep a master list of fileId's as we iterate over all the files in this Google Drive account.
    ; This master list will also be saved to the cache under the key "AllGoogleDriveFileIds".
    jsonMaster.i = CkJsonObject::ckCreate()
    If jsonMaster.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    jsonMasterArr.i = CkJsonArray::ckCreate()
    If jsonMasterArr.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkJsonObject::ckAppendArray2(jsonMaster,"fileIds",jsonMasterArr)

    ; Also keep a list of file paths.
    jsonMasterPaths.i = CkJsonArray::ckCreate()
    If jsonMasterPaths.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkJsonObject::ckAppendArray2(jsonMaster,"filePaths",jsonMasterPaths)

    ; The default page size is 100, with a max of 1000.
    CkRest::ckAddQueryParam(rest,"pageSize","200")

    json.i = CkJsonObject::ckCreate()
    If json.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    jsonFileMetadata.i
    i.i
    numFiles.i

    ; Send the request for the 1st page.
    jsonResponse.s = CkRest::ckFullRequestNoBody(rest,"GET","/drive/v3/files")

    pageNumber.i = 1
    pageToken.s
    bContinueLoop.i = CkRest::ckLastMethodSuccess(rest) AND (CkRest::ckResponseStatusCode(rest) = 200)

    While bContinueLoop = 1

        Debug "---- Page " + Str(pageNumber) + " ----"
        CkJsonObject::ckLoad(json,jsonResponse)

        numFiles = CkJsonObject::ckSizeOfArray(json,"files")
        i = 0
        While i < numFiles
            ; Add this file ID to the master list.
            CkJsonObject::setCkI(json, i)
            CkJsonArray::ckAddStringAt(jsonMasterArr,-1,CkJsonObject::ckStringOf(json,"files[i].id"))

            i = i + 1
        Wend

        ; Get the next page of files.
        ; If the "nextPageToken" is present in the JSON response, then use it in the "pageToken" parameter
        ; for the next request.   If no "nextPageToken" was present, then this was the last page of files.
        pageToken = CkJsonObject::ckStringOf(json,"nextPageToken")
        bContinueLoop = 0
        bHasMorePages.i = CkJsonObject::ckLastMethodSuccess(json)
        If bHasMorePages = 1
            CkRest::ckClearAllQueryParams(rest)
            CkRest::ckAddQueryParam(rest,"pageSize","200")
            CkRest::ckAddQueryParam(rest,"pageToken",pageToken)
            jsonResponse = CkRest::ckFullRequestNoBody(rest,"GET","/drive/v3/files")
            bContinueLoop = CkRest::ckLastMethodSuccess(rest) AND (CkRest::ckResponseStatusCode(rest) = 200)
            pageNumber = pageNumber + 1
        EndIf

    Wend

    ; Check to see if the above loop exited with errors...
    If CkRest::ckLastMethodSuccess(rest) = 0
        Debug CkRest::ckLastErrorText(rest)
        CkAuthGoogle::ckDispose(gAuth)
        CkRest::ckDispose(rest)
        CkCache::ckDispose(gdCache)
        CkDateTime::ckDispose(dtExpire)
        CkJsonObject::ckDispose(jsonMaster)
        CkJsonArray::ckDispose(jsonMasterArr)
        CkJsonArray::ckDispose(jsonMasterPaths)
        CkJsonObject::ckDispose(json)
        ProcedureReturn
    EndIf

    ; Check to see if the above loop exited with errors...
    ; A successful response will have a status code equal to 200.
    If CkRest::ckResponseStatusCode(rest) <> 200
        Debug "response status code = " + Str(CkRest::ckResponseStatusCode(rest))
        Debug "response status text = " + CkRest::ckResponseStatusText(rest)
        Debug "response header: " + CkRest::ckResponseHeader(rest)
        Debug "response JSON: " + jsonResponse
        CkAuthGoogle::ckDispose(gAuth)
        CkRest::ckDispose(rest)
        CkCache::ckDispose(gdCache)
        CkDateTime::ckDispose(dtExpire)
        CkJsonObject::ckDispose(jsonMaster)
        CkJsonArray::ckDispose(jsonMasterArr)
        CkJsonArray::ckDispose(jsonMasterPaths)
        CkJsonObject::ckDispose(json)
        ProcedureReturn
    EndIf

    ; Iterate over the file IDs and download the metadata for each, saving each to the cache...
    ; Also, keep in-memory hash entries of the name and parent[0] so we can quickly 
    ; build the path-->fileId cache entries. (Given that the Google Drive REST API uses
    ; fileIds, this gives us an easy way to lookup a fileId based on a filePath.)
    hashTable.i = CkHashtable::ckCreate()
    If hashTable.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    ; Set the capacity of the hash table to something reasonable for the number of files
    ; to be hashed.
    CkHashtable::ckClearWithNewCapacity(hashTable,521)

    sbPathForFileId.i = CkStringBuilder::ckCreate()
    If sbPathForFileId.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    ; Used for storing the file name and parents[0] in the hashTable.
    saFileInfo.i = CkStringArray::ckCreate()
    If saFileInfo.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkStringArray::setCkUnique(saFileInfo, 0)

    fileId.s
    numFiles = CkJsonObject::ckSizeOfArray(jsonMaster,"fileIds")
    i = 0
    While i < numFiles
        CkJsonObject::setCkI(jsonMaster, i)
        fileId = CkJsonObject::ckStringOf(jsonMaster,"fileIds[i]")
        CkStringBuilder::ckSetString(sbPathForFileId,"/drive/v3/files/")
        CkStringBuilder::ckAppend(sbPathForFileId,fileId)

        CkRest::ckClearAllQueryParams(rest)
        CkRest::ckAddQueryParam(rest,"fields",allFields)
        jsonResponse = CkRest::ckFullRequestNoBody(rest,"GET",CkStringBuilder::ckGetAsString(sbPathForFileId))
        If (CkRest::ckLastMethodSuccess(rest) <> 1) OR (CkRest::ckResponseStatusCode(rest) <> 200)
            ; Force an exit of this loop..
            numFiles = 0
        EndIf

        ; Save this file's metadata to the local cache.
        ; The lookup key is the fileId.
        CkCache::ckSaveTextDt(gdCache,fileId,dtExpire,"",jsonResponse)

        ; Get this file's name and parent[0], and put this information
        ; in our in-memory hashtable to be used below..
        CkJsonObject::ckLoad(json,jsonResponse)

        CkStringArray::ckClear(saFileInfo)
        CkStringArray::ckAppend(saFileInfo,CkJsonObject::ckStringOf(json,"name"))
        CkStringArray::ckAppend(saFileInfo,CkJsonObject::ckStringOf(json,"parents[0]"))
        CkHashtable::ckAddStr(hashTable,fileId,CkStringArray::ckSerialize(saFileInfo))

        Debug CkJsonObject::ckStringOf(json,"name") + ", " + CkJsonObject::ckStringOf(json,"parents[0]")

        i = i + 1
    Wend

    ; Check to see if the above loop exited with errors...
    If CkRest::ckLastMethodSuccess(rest) = 0
        Debug CkRest::ckLastErrorText(rest)
        CkAuthGoogle::ckDispose(gAuth)
        CkRest::ckDispose(rest)
        CkCache::ckDispose(gdCache)
        CkDateTime::ckDispose(dtExpire)
        CkJsonObject::ckDispose(jsonMaster)
        CkJsonArray::ckDispose(jsonMasterArr)
        CkJsonArray::ckDispose(jsonMasterPaths)
        CkJsonObject::ckDispose(json)
        CkHashtable::ckDispose(hashTable)
        CkStringBuilder::ckDispose(sbPathForFileId)
        CkStringArray::ckDispose(saFileInfo)
        ProcedureReturn
    EndIf

    ; Check to see if the above loop exited with errors...
    ; A successful response will have a status code equal to 200.
    If CkRest::ckResponseStatusCode(rest) <> 200
        Debug "response status code = " + Str(CkRest::ckResponseStatusCode(rest))
        Debug "response status text = " + CkRest::ckResponseStatusText(rest)
        Debug "response header: " + CkRest::ckResponseHeader(rest)
        Debug "response JSON: " + jsonResponse
        CkAuthGoogle::ckDispose(gAuth)
        CkRest::ckDispose(rest)
        CkCache::ckDispose(gdCache)
        CkDateTime::ckDispose(dtExpire)
        CkJsonObject::ckDispose(jsonMaster)
        CkJsonArray::ckDispose(jsonMasterArr)
        CkJsonArray::ckDispose(jsonMasterPaths)
        CkJsonObject::ckDispose(json)
        CkHashtable::ckDispose(hashTable)
        CkStringBuilder::ckDispose(sbPathForFileId)
        CkStringArray::ckDispose(saFileInfo)
        ProcedureReturn
    EndIf

    ; Now that all the fileId's are in the cache, let's build the directory path
    ; for each fileID.  

    ; (Technically, a fileId can have multiple parents, which means it can be in multiple directories
    ; at once.  This is only going to build directory paths following the 0'th parent ID in the parents list.)

    ; The directory path for files in "My Drive" will be just the filename.
    ; For files in sub-directories, the path will be relative, such as "subdir1/subdir2/something.pdf"
    ; 

    Debug "---- building paths ----"

    sbPath.i = CkStringBuilder::ckCreate()
    If sbPath.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    numFiles = CkJsonObject::ckSizeOfArray(jsonMaster,"fileIds")
    i = 0
    While i < numFiles
        CkJsonObject::setCkI(jsonMaster, i)

        CkStringBuilder::ckClear(sbPath)

        fileId = CkJsonObject::ckStringOf(jsonMaster,"fileIds[i]")
        bFinished.i = 0
        While (bFinished = 0)
            CkStringArray::ckClear(saFileInfo)
            CkStringArray::ckAppendSerialized(saFileInfo,CkHashtable::ckLookupStr(hashTable,fileId))
            ; Append this file or directory name.
            CkStringBuilder::ckPrepend(sbPath,CkStringArray::ckGetString(saFileInfo,0))
            ; Get the parent fileId
            fileId = CkStringArray::ckGetString(saFileInfo,1)
            ; If this fileId is not in the hashtable, then it's the fileId for "My Drive", and we are finished.
            If CkHashtable::ckContains(hashTable,fileId) = 0
                bFinished = 1
            Else
                CkStringBuilder::ckPrepend(sbPath,"/")
            EndIf

        Wend

        Debug Str(i) + ": " + CkStringBuilder::ckGetAsString(sbPath)

        ; Store the filePath --> fileId mapping in our local cache.
        fileId = CkJsonObject::ckStringOf(jsonMaster,"fileIds[i]")
        CkCache::ckSaveTextDt(gdCache,CkStringBuilder::ckGetAsString(sbPath),dtExpire,"",fileId)

        CkJsonArray::ckAddStringAt(jsonMasterPaths,-1,CkStringBuilder::ckGetAsString(sbPath))

        i = i + 1
    Wend

    ; Save the master list of file IDs and file paths to the local cache.
    CkJsonObject::setCkEmitCompact(jsonMaster, 0)
    strJsonMaster.s = CkJsonObject::ckEmit(jsonMaster)
    CkCache::ckSaveTextNoExpire(gdCache,"AllGoogleDriveFileIds","",strJsonMaster)
    Debug "JSON Master Record:"
    Debug strJsonMaster

    ; The JSON Master Cache Record looks something like this:
    ; An application can load the JSON master record and iterate over all the files
    ; in Google Drive by file ID, or by path.  
    ; {
    ;   "fileIds": [
    ;     "0B53Q6OSTWYolQlExSlBQT1phZXM",
    ;     "0B53Q6OSTWYolVHRPVkxtYWFtZkk",
    ;     "0B53Q6OSTWYolRGZEV3ZGUTZfNFk",
    ;     "0B53Q6OSTWYolS2FXSjliMXQxSU0",
    ;     "0B53Q6OSTWYolZUhxckMzb0dRMzg",
    ;     "0B53Q6OSTWYolbUF6WS1Gei1oalk",
    ;     "0B53Q6OSTWYola296ODZUSm5GYU0",
    ;     "0B53Q6OSTWYolbTE3c3J5RHBUcHM",
    ;     "0B53Q6OSTWYolTmhybWJSUGd5Q2c",
    ;     "0B53Q6OSTWYolY2tPU1BnYW02T2c",
    ;     "0B53Q6OSTWYolTTBBR2NvUE81Zzg",
    ;   ],
    ;   "filePaths": [
    ;     "testFolder/abc/123/pigs.json",
    ;     "testFolder/starfish20.jpg",
    ;     "testFolder/penguins2.jpg",
    ;     "testFolder/starfish.jpg",
    ;     "testFolder/abc/123/starfish.jpg",
    ;     "testFolder/abc/123/penguins.jpg",
    ;     "testFolder/abc/123",
    ;     "testFolder/abc",
    ;     "testFolder/testHello.txt",
    ;     "testFolder",
    ;     "helloWorld.txt",
    ;   ]
    ; }

    Debug "Entire cache rebuilt..."


    CkAuthGoogle::ckDispose(gAuth)
    CkRest::ckDispose(rest)
    CkCache::ckDispose(gdCache)
    CkDateTime::ckDispose(dtExpire)
    CkJsonObject::ckDispose(jsonMaster)
    CkJsonArray::ckDispose(jsonMasterArr)
    CkJsonArray::ckDispose(jsonMasterPaths)
    CkJsonObject::ckDispose(json)
    CkHashtable::ckDispose(hashTable)
    CkStringBuilder::ckDispose(sbPathForFileId)
    CkStringArray::ckDispose(saFileInfo)
    CkStringBuilder::ckDispose(sbPath)


    ProcedureReturn
EndProcedure