Synchronize Gmail message IDs and labels between runs
See more GMail REST API Examples
Maintain gmail-sync.json containing the mailbox address, a string historyId, and a small local cache of message IDs, thread IDs, and labels. The first run scans the complete mailbox including Spam and Trash. Later runs follow all history pages and refresh only messages affected by additions, deletions, or label changes. A message GET returning 404 removes that message from the local cache.
An expired history checkpoint causes a full rebuild. The full scan records a checkpoint before listing messages, so the next run also reconciles changes made during the scan. Incremental runs save the final history response checkpoint only after every affected message has been processed. The cache and checkpoint are committed together, with a previous-file recovery path. History IDs remain strings throughout.
Run in a private directory and serialize runs sharing the cache. The sample holds the cache in memory for clarity. Larger applications can use database upserts and deletions, committing the checkpoint in the same transaction. The cache contains message identifiers and labels. Fetch MIME or selected headers when the application needs message content.
Authorize the application with https://www.googleapis.com/auth/gmail.readonly. The example reads the access token from gmail-token.json in a private working directory. Authorize a desktop application.
v11.4.0 or greater.
var success = false;
// Assumes Chilkat is unlocked. See Global Unlock Sample.
var token = new CkJsonObject();
success = token.LoadFile("gmail-token.json");
if (success !== true) {
console.log("Could not read gmail-token.json from the private working directory.");
return;
}
if ((token.JsonTypeOf("access_token") !== 1) || (token.StringOfEquals("access_token","",true) == true)) {
console.log("An OAuth access token is required.");
return;
}
var http = new CkHttp();
http.RequireSslCertVerify = true;
http.AuthToken = token.StringOf("access_token");
var resp = new CkHttpResponse();
var reply = new CkJsonObject();
var files = new CkFileAccess();
// Recover the previous complete file if a run stopped during replacement.
var fileStatus = files.FileExists3("gmail-sync.json");
if (fileStatus < 0) {
console.log("Could not inspect gmail-sync.json.");
return;
}
if (fileStatus == 0) {
var backupStatus = files.FileExists3("gmail-sync.json.previous");
if (backupStatus < 0) {
console.log("Could not inspect the backup file.");
return;
}
if (backupStatus == 1) {
success = files.FileRename("gmail-sync.json.previous","gmail-sync.json");
if (success !== true) {
console.log("Could not restore the previous complete file.");
return;
}
fileStatus = 1;
}
}
var state = new CkJsonObject();
if (fileStatus == 1) {
success = state.LoadFile("gmail-sync.json");
if (success !== true) {
console.log("Could not read the existing synchronization state.");
return;
}
if ((state.JsonTypeOf("historyId") !== 1) || (state.JsonTypeOf("messages") !== 3) || (state.JsonTypeOf("emailAddress") !== 1)) {
console.log("The synchronization state is incomplete. Restore its previous complete copy.");
return;
}
}
success = http.HttpNoBody("GET","https://gmail.googleapis.com/gmail/v1/users/me/profile",resp);
if (resp.StatusCode == 0) {
console.log("Gmail did not return an HTTP response.");
return;
}
if ((success !== true) || (resp.StatusCode < 200) || (resp.StatusCode >= 300)) {
console.log("Gmail HTTP " + resp.StatusCode);
success = reply.Load(resp.BodyStr);
if (success == true) {
console.log(reply.StringOf("error.message"));
}
return;
}
success = reply.Load(resp.BodyStr);
if (success !== true) {
console.log("Expected a Gmail JSON response.");
return;
}
if ((reply.JsonTypeOf("emailAddress") !== 1) || (reply.StringOfEquals("emailAddress","",true) == true)) {
console.log("Expected emailAddress in the response.");
return;
}
if ((reply.JsonTypeOf("historyId") !== 1) || (reply.StringOfEquals("historyId","",true) == true)) {
console.log("Expected historyId in the response.");
return;
}
var mailbox = reply.StringOf("emailAddress");
var baseline = reply.StringOf("historyId");
if (fileStatus == 1) {
if (state.StringOfEquals("emailAddress",mailbox,false) !== true) {
console.log("Use a separate synchronization file for this mailbox.");
return;
}
}
var fullSync = fileStatus == 0;
var checkpoint = baseline;
var ids = new CkStringArray();
ids.Unique = true;
var baseUrl = new CkStringBuilder();
var url = new CkStringBuilder();
var pageToken = "";
var more = true;
if (fullSync !== true) {
baseUrl.SetString("https://gmail.googleapis.com/gmail/v1/users/me/history?maxResults=500&startHistoryId=");
baseUrl.Append(http.UrlEncode(state.StringOf("historyId")));
var changeTypes = new CkStringTable();
success = changeTypes.Append("messagesAdded");
if (success !== true) {
console.log(changeTypes.LastErrorText);
return;
}
success = changeTypes.Append("messagesDeleted");
if (success !== true) {
console.log(changeTypes.LastErrorText);
return;
}
success = changeTypes.Append("labelsAdded");
if (success !== true) {
console.log(changeTypes.LastErrorText);
return;
}
success = changeTypes.Append("labelsRemoved");
if (success !== true) {
console.log(changeTypes.LastErrorText);
return;
}
var record = new CkJsonObject();
var eventPath = new CkStringBuilder();
while ((more == true)) {
url.SetString(baseUrl.GetAsString());
url.Append("&pageToken=");
url.Append(http.UrlEncode(pageToken));
success = http.HttpNoBody("GET",url.GetAsString(),resp);
if (resp.StatusCode == 404) {
console.log("History checkpoint expired. Rebuilding the cache.");
fullSync = true;
more = false;
}
else {
if (resp.StatusCode == 0) {
console.log("Gmail did not return an HTTP response.");
return;
}
if ((success !== true) || (resp.StatusCode < 200) || (resp.StatusCode >= 300)) {
console.log("Gmail HTTP " + resp.StatusCode);
success = reply.Load(resp.BodyStr);
if (success == true) {
console.log(reply.StringOf("error.message"));
}
return;
}
success = reply.Load(resp.BodyStr);
if (success !== true) {
console.log("Expected a Gmail JSON response.");
return;
}
var h = 0;
while ((h < reply.SizeOfArray("history"))) {
reply.I = h;
success = reply.ObjectOf2("history[i]",record);
if (success !== true) {
console.log(reply.LastErrorText);
return;
}
var t = 0;
while ((t < changeTypes.Count)) {
var j = 0;
while ((j < record.SizeOfArray(changeTypes.StringAt(t)))) {
eventPath.SetString(changeTypes.StringAt(t));
eventPath.Append("[");
eventPath.AppendInt(j);
eventPath.Append("].message.id");
success = ids.Append(record.StringOf(eventPath.GetAsString()));
if (success !== true) {
console.log(ids.LastErrorText);
return;
}
j = j+1;
}
t = t+1;
}
h = h+1;
}
pageToken = "";
if (reply.JsonTypeOf("nextPageToken") == 1) {
pageToken = reply.StringOf("nextPageToken");
}
more = pageToken !== "";
if (more !== true) {
if ((reply.JsonTypeOf("historyId") !== 1) || (reply.StringOfEquals("historyId","",true) == true)) {
console.log("Expected historyId in the response.");
return;
}
checkpoint = reply.StringOf("historyId");
}
}
}
}
if (fullSync == true) {
// Rebuild in memory. Any failure leaves the committed file available for a retry.
ids.Clear();
state.Clear();
success = state.UpdateString("emailAddress",mailbox);
if (success !== true) {
console.log(state.LastErrorText);
return;
}
success = state.UpdateNewObject("messages");
if (success !== true) {
console.log(state.LastErrorText);
return;
}
checkpoint = baseline;
pageToken = "";
more = true;
while ((more == true)) {
url.SetString("https://gmail.googleapis.com/gmail/v1/users/me/messages?maxResults=500&includeSpamTrash=true&pageToken=");
url.Append(http.UrlEncode(pageToken));
success = http.HttpNoBody("GET",url.GetAsString(),resp);
if (resp.StatusCode == 0) {
console.log("Gmail did not return an HTTP response.");
return;
}
if ((success !== true) || (resp.StatusCode < 200) || (resp.StatusCode >= 300)) {
console.log("Gmail HTTP " + resp.StatusCode);
success = reply.Load(resp.BodyStr);
if (success == true) {
console.log(reply.StringOf("error.message"));
}
return;
}
success = reply.Load(resp.BodyStr);
if (success !== true) {
console.log("Expected a Gmail JSON response.");
return;
}
var m = 0;
while ((m < reply.SizeOfArray("messages"))) {
reply.I = m;
success = ids.Append(reply.StringOf("messages[i].id"));
if (success !== true) {
console.log(ids.LastErrorText);
return;
}
m = m+1;
}
pageToken = "";
if (reply.JsonTypeOf("nextPageToken") == 1) {
pageToken = reply.StringOf("nextPageToken");
}
more = pageToken !== "";
}
}
var messages = new CkJsonObject();
success = state.ObjectOf2("messages",messages);
if (success !== true) {
console.log(state.LastErrorText);
return;
}
var cachedMessage = new CkJsonObject();
var i = 0;
while ((i < ids.Count)) {
var messageId = ids.GetString(i);
url.SetString("https://gmail.googleapis.com/gmail/v1/users/me/messages/");
url.Append(http.UrlEncode(messageId));
url.Append("?format=minimal&fields=id,threadId,labelIds");
success = http.HttpNoBody("GET",url.GetAsString(),resp);
if (resp.StatusCode == 404) {
if (messages.HasMember(messageId) == true) {
success = messages.Delete(messageId);
if (success !== true) {
console.log(messages.LastErrorText);
return;
}
}
console.log("Removed from cache: " + messageId);
}
else {
if (resp.StatusCode == 0) {
console.log("Gmail did not return an HTTP response.");
return;
}
if ((success !== true) || (resp.StatusCode < 200) || (resp.StatusCode >= 300)) {
console.log("Gmail HTTP " + resp.StatusCode);
success = reply.Load(resp.BodyStr);
if (success == true) {
console.log(reply.StringOf("error.message"));
}
return;
}
success = reply.Load(resp.BodyStr);
if (success !== true) {
console.log("Expected a Gmail JSON response.");
return;
}
if (reply.StringOfEquals("id",messageId,true) !== true) {
console.log("Unexpected message ID in the response.");
return;
}
success = messages.UpdateNewObject(messageId);
if (success !== true) {
console.log(messages.LastErrorText);
return;
}
success = messages.ObjectOf2(messageId,cachedMessage);
if (success !== true) {
console.log(messages.LastErrorText);
return;
}
success = cachedMessage.Load(reply.Emit());
if (success !== true) {
console.log(cachedMessage.LastErrorText);
return;
}
console.log("Cached message: " + messageId);
}
i = i+1;
}
success = state.UpdateString("historyId",checkpoint);
if (success !== true) {
console.log(state.LastErrorText);
return;
}
// Prepare the complete replacement before moving the last committed file.
success = state.WriteFile("gmail-sync.json.new");
if (success !== true) {
console.log("Could not write the replacement file.");
return;
}
if (files.FileExists("gmail-sync.json") == true) {
if (files.FileExists("gmail-sync.json.previous") == true) {
success = files.FileDelete("gmail-sync.json.previous");
if (success !== true) {
console.log("Could not rotate the previous backup.");
return;
}
}
success = files.FileRename("gmail-sync.json","gmail-sync.json.previous");
if (success !== true) {
console.log("Could not retain the previous complete file.");
return;
}
}
success = files.FileRename("gmail-sync.json.new","gmail-sync.json");
if (success !== true) {
console.log("Could not commit the replacement. The previous complete file is retained.");
return;
}
console.log("Synchronization committed. Cached messages: " + messages.Size);
console.log("Saved history ID: " + checkpoint);