Line data Source code
1 : #include "gmail_sync.h"
2 : #include "gmail_client.h"
3 : #include "local_store.h"
4 : #include "mail_rules.h"
5 : #include "mime_util.h"
6 : #include "json_util.h"
7 : #include "logger.h"
8 : #include "raii.h"
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <string.h>
12 : #include <stddef.h>
13 :
14 : /* ── Progress callbacks ───────────────────────────────────────────── */
15 :
16 : /* Called by gmail_list_messages after each page: cur = messages collected so far */
17 27 : static void list_progress_cb(size_t cur, size_t total, void *ctx) {
18 : (void)total; (void)ctx;
19 27 : fprintf(stderr, "\r\033[K Listing messages... %zu found", cur);
20 27 : fflush(stderr);
21 27 : }
22 :
23 : /* ── Gmail .hdr file format ───────────────────────────────────────── */
24 :
25 : /**
26 : * Build a .hdr string from raw message headers and label list.
27 : * Format: from\tsubject\tdate\tlabel1,label2,...\tflags\n
28 : *
29 : * Returns heap-allocated string. Caller must free().
30 : */
31 601 : char *gmail_sync_build_hdr(const char *raw_msg, char **labels, int label_count) {
32 1202 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
33 1202 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
34 1202 : RAII_STRING char *date_raw = mime_get_header(raw_msg, "Date");
35 :
36 1202 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
37 1202 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
38 601 : RAII_STRING char *date_fmt = date_raw ? mime_format_date(date_raw) : NULL;
39 :
40 601 : const char *from = from_dec ? from_dec : "";
41 601 : const char *subj = subj_dec ? subj_dec : "";
42 601 : const char *date = date_fmt ? date_fmt : "";
43 :
44 : /* Build comma-separated label string */
45 601 : size_t lbl_len = 1;
46 1573 : for (int i = 0; i < label_count; i++)
47 972 : lbl_len += strlen(labels[i]) + 1;
48 601 : char *lbl_str = calloc(lbl_len, 1);
49 601 : if (lbl_str) {
50 1573 : for (int i = 0; i < label_count; i++) {
51 972 : if (i > 0) strcat(lbl_str, ",");
52 972 : strcat(lbl_str, labels[i]);
53 : }
54 : }
55 :
56 : /* Compute flags bitmask from labels */
57 601 : int flags = 0;
58 1573 : for (int i = 0; i < label_count; i++) {
59 972 : if (strcmp(labels[i], "UNREAD") == 0) flags |= MSG_FLAG_UNSEEN;
60 972 : if (strcmp(labels[i], "STARRED") == 0) flags |= MSG_FLAG_FLAGGED;
61 972 : if (strcmp(labels[i], "SPAM") == 0) flags |= MSG_FLAG_JUNK;
62 : }
63 :
64 : /* Replace tabs in fields with spaces */
65 601 : char *hdr = NULL;
66 601 : if (asprintf(&hdr, "%s\t%s\t%s\t%s\t%d",
67 : from, subj, date, lbl_str ? lbl_str : "", flags) == -1)
68 0 : hdr = NULL;
69 601 : free(lbl_str);
70 :
71 : /* Sanitise: replace any tabs within field values */
72 601 : if (hdr) {
73 : /* The first 4 tabs are field separators; tabs within values got
74 : * inserted by asprintf if field values contained tabs. Since we
75 : * used tab as separator this is inherently safe (mime_decode_words
76 : * doesn't produce tabs), but defend anyway. */
77 : }
78 :
79 601 : return hdr;
80 : }
81 :
82 : /* ── Filtered labels (metadata-only, excluded from indexing) ──────── */
83 :
84 6557 : int gmail_sync_is_filtered_label(const char *label_id) {
85 6557 : if (!label_id) return 1;
86 6557 : if (strcmp(label_id, "IMPORTANT") == 0) return 1;
87 6557 : if (strcmp(label_id, "CHAT") == 0) return 1;
88 6557 : return 0;
89 : }
90 :
91 : /* Returns 1 if label_id is a Gmail automatic inbox category (CATEGORY_*).
92 : * Category labels are indexed like user labels, but a message whose ONLY
93 : * non-filtered labels are CATEGORY_* is also added to _nolabel (Archive). */
94 6553 : static int is_category_label(const char *label_id) {
95 6553 : return label_id && strncmp(label_id, "CATEGORY_", 9) == 0;
96 : }
97 :
98 : /* ── Mail rules helper ───────────────────────────────────────────── */
99 :
100 : /* Apply mail rules to a newly stored message.
101 : * Builds labels_csv from Gmail label IDs (resolving user labels to names),
102 : * calls mail_rules_apply(), then updates the local .hdr and label indexes. */
103 601 : static void apply_rules_to_new_message(const MailRules *rules, const char *uid,
104 : const char *raw_msg,
105 : char **labels, int label_count)
106 : {
107 601 : if (!rules || rules->count == 0) return;
108 :
109 0 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
110 0 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
111 0 : RAII_STRING char *to_raw = mime_get_header(raw_msg, "To");
112 0 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
113 0 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
114 0 : RAII_STRING char *to_dec = to_raw ? mime_decode_words(to_raw) : NULL;
115 :
116 : /* Build labels_csv using friendly names where available */
117 0 : size_t lcsz = 1;
118 0 : for (int i = 0; i < label_count; i++) {
119 0 : char *name = local_gmail_label_name_lookup(labels[i]);
120 0 : lcsz += strlen(name ? name : labels[i]) + 2;
121 0 : free(name);
122 : }
123 0 : char *lcsv = calloc(lcsz, 1);
124 0 : if (!lcsv) return;
125 0 : for (int i = 0; i < label_count; i++) {
126 0 : char *name = local_gmail_label_name_lookup(labels[i]);
127 0 : const char *display = name ? name : labels[i];
128 0 : if (lcsv[0]) strcat(lcsv, ",");
129 0 : strcat(lcsv, display);
130 0 : free(name);
131 : }
132 :
133 0 : char **add_out = NULL; int add_count = 0;
134 0 : char **rm_out = NULL; int rm_count = 0;
135 0 : int fired = mail_rules_apply(rules,
136 : from_dec, subj_dec, to_dec, lcsv,
137 : NULL, (time_t)0, /* body/date unavailable during Gmail sync */
138 : &add_out, &add_count,
139 : &rm_out, &rm_count);
140 0 : free(lcsv);
141 0 : if (fired <= 0) return;
142 :
143 0 : logger_log(LOG_INFO, "gmail_sync: rules fired=%d for %s (add=%d rm=%d)",
144 : fired, uid, add_count, rm_count);
145 :
146 : /* Update local .hdr and label indexes */
147 0 : local_hdr_update_labels("", uid,
148 : (const char **)add_out, add_count,
149 : (const char **)rm_out, rm_count);
150 0 : for (int i = 0; i < add_count; i++) {
151 0 : label_idx_add(add_out[i], uid);
152 0 : free(add_out[i]);
153 : }
154 0 : for (int i = 0; i < rm_count; i++) {
155 0 : label_idx_remove(rm_out[i], uid);
156 0 : free(rm_out[i]);
157 : }
158 0 : free(add_out);
159 0 : free(rm_out);
160 :
161 : /* Update contact suggestion cache */
162 : {
163 0 : char *from_h = mime_get_header(raw_msg, "From");
164 0 : char *to_h = mime_get_header(raw_msg, "To");
165 0 : char *cc_h = mime_get_header(raw_msg, "Cc");
166 0 : local_contacts_update(from_h, to_h, cc_h);
167 0 : free(from_h); free(to_h); free(cc_h);
168 : }
169 : }
170 :
171 : /* ── Label index rebuild ──────────────────────────────────────────── */
172 :
173 : typedef struct { char label[64]; char uid[17]; } LabelUidPair;
174 :
175 38049 : static int cmp_lbl_uid_pair(const void *a, const void *b) {
176 38049 : const LabelUidPair *pa = a, *pb = b;
177 38049 : int c = strcmp(pa->label, pb->label);
178 38049 : return c ? c : strcmp(pa->uid, pb->uid);
179 : }
180 :
181 : /**
182 : * Rebuild ALL label .idx files from the .hdr files for the given UIDs.
183 : *
184 : * Efficient O(N log N) approach:
185 : * 1. Read each .hdr and collect (label, uid) pairs in memory.
186 : * 2. Sort the flat pair array.
187 : * 3. Write each label's .idx file in one grouped pass.
188 : *
189 : * This is called at the end of every full sync so that cached messages
190 : * (whose .idx entries were never written) are correctly indexed.
191 : */
192 27 : static void rebuild_label_indexes(const char (*uids)[17], int uid_count) {
193 27 : if (uid_count <= 0) return;
194 :
195 27 : fprintf(stderr, " Rebuilding label indexes...");
196 27 : fflush(stderr);
197 :
198 : /* Phase 1: collect (label, uid) pairs from all .hdr files */
199 27 : size_t cap = (size_t)uid_count * 5; /* ~5 labels per message */
200 27 : LabelUidPair *pairs = malloc(cap * sizeof(LabelUidPair));
201 27 : if (!pairs) {
202 0 : fprintf(stderr, " [out of memory]\n");
203 0 : return;
204 : }
205 27 : int npairs = 0;
206 :
207 3595 : for (int i = 0; i < uid_count; i++) {
208 : /* Load full .hdr so we can both collect label pairs and sync the
209 : * flags integer in one read (avoid a separate local_hdr_get_labels
210 : * call followed by local_hdr_update_labels). */
211 3568 : char *hdr = local_hdr_load("", uids[i]);
212 3568 : if (!hdr) continue;
213 :
214 : /* Locate labels field (4th tab-separated token).
215 : * Track the tab pointer so we can NUL-terminate the prefix later. */
216 3568 : char *t3_tab = hdr;
217 14272 : for (int f = 0; f < 3; f++) {
218 10704 : t3_tab = strchr(t3_tab, '\t');
219 10704 : if (!t3_tab) break;
220 10704 : if (f < 2) t3_tab++;
221 : }
222 3568 : if (!t3_tab || t3_tab == hdr) { free(hdr); continue; }
223 3568 : char *lbl_start = t3_tab + 1; /* start of labels CSV field */
224 :
225 : /* Locate optional flags field (5th token) and read old value */
226 3568 : char *t4 = strchr(lbl_start, '\t');
227 3568 : int old_flags = 0;
228 3568 : if (t4) {
229 3568 : old_flags = atoi(t4 + 1);
230 3568 : *t4 = '\0'; /* NUL-terminate labels field in-place */
231 : } else {
232 0 : char *nl = strchr(lbl_start, '\n');
233 0 : if (nl) *nl = '\0';
234 : }
235 :
236 : /* Derive new flags from labels (preserving non-label bits) */
237 3568 : int new_flags = old_flags & ~(MSG_FLAG_UNSEEN | MSG_FLAG_FLAGGED);
238 3568 : int has_real = 0;
239 :
240 : /* Iterate labels via a copy (tokenising modifies the string) */
241 3568 : char *lbl_copy = strdup(lbl_start);
242 3568 : if (!lbl_copy) { free(hdr); continue; }
243 :
244 3568 : char *tok = lbl_copy;
245 9156 : while (tok) {
246 5588 : char *comma = strchr(tok, ',');
247 5588 : if (comma) *comma = '\0';
248 :
249 5588 : if (tok[0] && !gmail_sync_is_filtered_label(tok)) {
250 5585 : const char *idx_name = tok;
251 5585 : if (strcmp(tok, "SPAM") == 0) idx_name = "_spam";
252 5585 : else if (strcmp(tok, "TRASH") == 0) idx_name = "_trash";
253 :
254 5585 : if (npairs >= (int)cap) {
255 0 : cap = cap * 2 + 1;
256 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
257 0 : if (!tmp) { free(lbl_copy); free(hdr); free(pairs); return; }
258 0 : pairs = tmp;
259 : }
260 5585 : strncpy(pairs[npairs].label, idx_name, 63);
261 5585 : pairs[npairs].label[63] = '\0';
262 5585 : strncpy(pairs[npairs].uid, uids[i], 16);
263 5585 : pairs[npairs].uid[16] = '\0';
264 5585 : npairs++;
265 :
266 5585 : if (!is_category_label(tok)) has_real = 1;
267 5585 : if (strcmp(tok, "UNREAD") == 0) new_flags |= MSG_FLAG_UNSEEN;
268 5585 : if (strcmp(tok, "STARRED") == 0) new_flags |= MSG_FLAG_FLAGGED;
269 : }
270 5588 : tok = comma ? comma + 1 : NULL;
271 : }
272 3568 : free(lbl_copy);
273 :
274 : /* Messages with no real (non-CATEGORY_) label → Archive.
275 : * Archived messages are always considered read. */
276 3568 : if (!has_real) {
277 3 : if (npairs >= (int)cap) {
278 0 : cap = cap * 2 + 1;
279 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
280 0 : if (!tmp) { free(hdr); free(pairs); return; }
281 0 : pairs = tmp;
282 : }
283 3 : strncpy(pairs[npairs].label, "_nolabel", 63);
284 3 : strncpy(pairs[npairs].uid, uids[i], 16);
285 3 : pairs[npairs].uid[16] = '\0';
286 3 : npairs++;
287 3 : new_flags &= ~MSG_FLAG_UNSEEN;
288 : }
289 :
290 : /* Sync flags integer if it disagrees with the labels CSV.
291 : * lbl_start still points into hdr at the labels field.
292 : * NUL-terminate the prefix at t3_tab then reassemble. */
293 3568 : if (new_flags != old_flags) {
294 0 : *t3_tab = '\0';
295 0 : char *updated = NULL;
296 0 : if (asprintf(&updated, "%s\t%s\t%d", hdr, lbl_start, new_flags) != -1) {
297 0 : local_hdr_save("", uids[i], updated, strlen(updated));
298 0 : free(updated);
299 : }
300 : }
301 :
302 3568 : free(hdr);
303 : }
304 :
305 : /* Phase 2: sort by (label, uid) */
306 27 : qsort(pairs, (size_t)npairs, sizeof(LabelUidPair), cmp_lbl_uid_pair);
307 :
308 : /* Phase 3: group by label and write each .idx file */
309 27 : int labels_written = 0;
310 27 : int i = 0;
311 136 : while (i < npairs) {
312 109 : const char *cur_label = pairs[i].label;
313 109 : int j = i;
314 5697 : while (j < npairs && strcmp(pairs[j].label, cur_label) == 0) j++;
315 109 : int run = j - i;
316 :
317 109 : char (*uid_arr)[17] = malloc((size_t)run * sizeof(char[17]));
318 109 : if (uid_arr) {
319 109 : int unique = 0;
320 5697 : for (int k = i; k < j; k++) {
321 5588 : if (unique == 0 ||
322 5479 : strcmp(uid_arr[unique - 1], pairs[k].uid) != 0) {
323 5588 : memcpy(uid_arr[unique++], pairs[k].uid, 17);
324 : }
325 : }
326 109 : label_idx_write(cur_label, (const char (*)[17])uid_arr, unique);
327 109 : free(uid_arr);
328 109 : labels_written++;
329 : }
330 109 : i = j;
331 : }
332 27 : free(pairs);
333 :
334 27 : fprintf(stderr, "\r\033[K Label indexes rebuilt (%d labels)\n",
335 : labels_written);
336 27 : logger_log(LOG_INFO,
337 : "gmail_sync: rebuilt %d label indexes from %d messages",
338 : labels_written, uid_count);
339 : }
340 :
341 : /**
342 : * Rebuild all label .idx files from locally cached .hdr files.
343 : * Does NOT contact the Gmail API.
344 : * Use this to repair missing or incomplete indexes without re-downloading.
345 : */
346 9 : int gmail_sync_rebuild_indexes(void) {
347 9 : char (*uids)[17] = NULL;
348 9 : int count = 0;
349 9 : if (local_hdr_list_all_uids("", &uids, &count) != 0) {
350 0 : fprintf(stderr, "Error: could not scan local message store.\n");
351 0 : return -1;
352 : }
353 9 : fprintf(stderr, " Found %d cached messages.\n", count);
354 9 : rebuild_label_indexes((const char (*)[17])uids, count);
355 9 : free(uids);
356 9 : return 0;
357 : }
358 :
359 : /* ── Single message fetch+store helper ───────────────────────────────── */
360 :
361 : /**
362 : * Fetch one message from the Gmail API, save .eml + .hdr, apply rules,
363 : * update label indexes.
364 : * Returns 0 on success, -1 on fetch error (transient; caller should retry).
365 : */
366 599 : static int store_fetched_message(GmailClient *gc, const char *uid,
367 : const MailRules *rules)
368 : {
369 599 : char **labels = NULL;
370 599 : int label_count = 0;
371 599 : char *raw = gmail_fetch_message(gc, uid, &labels, &label_count);
372 599 : if (!raw) {
373 0 : logger_log(LOG_WARN, "gmail_sync: failed to fetch %s", uid);
374 0 : for (int j = 0; j < label_count; j++) free(labels[j]);
375 0 : free(labels);
376 0 : return -1;
377 : }
378 :
379 599 : local_msg_save("", uid, raw, strlen(raw));
380 :
381 599 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
382 599 : if (hdr) { local_hdr_save("", uid, hdr, strlen(hdr)); free(hdr); }
383 :
384 599 : apply_rules_to_new_message(rules, uid, raw, labels, label_count);
385 599 : free(raw);
386 :
387 599 : int has_real_label = 0;
388 1567 : for (int j = 0; j < label_count; j++) {
389 968 : if (gmail_sync_is_filtered_label(labels[j])) continue;
390 968 : const char *idx_name = labels[j];
391 968 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
392 968 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
393 968 : label_idx_add(idx_name, uid);
394 968 : if (!is_category_label(labels[j])) has_real_label = 1;
395 : }
396 599 : if (!has_real_label) {
397 0 : label_idx_add("_nolabel", uid);
398 0 : int cur_flags = 0;
399 0 : for (int j = 0; j < label_count; j++) {
400 0 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
401 0 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
402 : }
403 0 : if (cur_flags & MSG_FLAG_UNSEEN)
404 0 : local_hdr_update_flags("", uid, cur_flags & ~MSG_FLAG_UNSEEN);
405 : }
406 :
407 1567 : for (int j = 0; j < label_count; j++) free(labels[j]);
408 599 : free(labels);
409 599 : return 0;
410 : }
411 :
412 : /* ── Reconcile: discover missing UIDs and queue them ─────────────────── */
413 :
414 : /**
415 : * List all server-side message IDs, compare with the local store, and
416 : * add any missing UIDs to pending_fetch.tsv. Does NOT download messages.
417 : *
418 : * Also updates the historyId and label name mapping so subsequent
419 : * incremental syncs know where to resume from.
420 : *
421 : * Returns the number of UIDs added to the pending-fetch queue, or -1 on
422 : * a fatal error (e.g. the server cannot be reached).
423 : */
424 18 : int gmail_sync_reconcile(GmailClient *gc) {
425 18 : logger_log(LOG_INFO, "gmail_sync: reconcile — listing server messages");
426 :
427 18 : fprintf(stderr, " Listing messages...");
428 18 : fflush(stderr);
429 18 : gmail_set_progress(gc, list_progress_cb, NULL);
430 :
431 18 : char (*all_uids)[17] = NULL;
432 18 : int uid_count = 0;
433 18 : char *list_history_id = NULL;
434 18 : if (gmail_list_messages(gc, NULL, NULL, &all_uids, &uid_count, &list_history_id) != 0) {
435 0 : gmail_set_progress(gc, NULL, NULL);
436 0 : free(list_history_id);
437 0 : logger_log(LOG_ERROR, "gmail_sync: reconcile failed to list messages");
438 0 : return -1;
439 : }
440 18 : gmail_set_progress(gc, NULL, NULL);
441 18 : fprintf(stderr, "\r\033[K %d messages on server\n", uid_count);
442 :
443 : /* Clear any stale pending_fetch entries before repopulating */
444 18 : local_pending_fetch_clear();
445 :
446 18 : int queued = 0, cached = 0;
447 1836 : for (int i = 0; i < uid_count; i++) {
448 1818 : const char *uid = all_uids[i];
449 1818 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
450 1219 : cached++;
451 1219 : if (i % 500 == 0 || i == uid_count - 1) {
452 18 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
453 : i + 1, uid_count);
454 18 : fflush(stderr);
455 : }
456 1219 : continue;
457 : }
458 599 : local_pending_fetch_add(uid);
459 599 : queued++;
460 599 : if ((cached + queued) % 500 == 0 || i == uid_count - 1) {
461 9 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
462 : i + 1, uid_count);
463 9 : fflush(stderr);
464 : }
465 : }
466 18 : if (uid_count > 0)
467 18 : fprintf(stderr, "\r\033[K %d cached, %d queued for download\n",
468 : cached, queued);
469 :
470 : /* Save historyId so next run can use incremental sync.
471 : * Prefer the historyId from the messages.list response (always fresh);
472 : * fall back to the /profile endpoint only if that field was absent. */
473 18 : if (list_history_id) {
474 18 : fprintf(stderr, " historyId from list response: %s\n", list_history_id);
475 18 : local_gmail_history_save(list_history_id);
476 18 : free(list_history_id);
477 18 : list_history_id = NULL;
478 : } else {
479 0 : RAII_STRING char *hid = gmail_get_history_id(gc);
480 0 : if (hid)
481 0 : local_gmail_history_save(hid);
482 : else
483 0 : logger_log(LOG_WARN, "gmail_sync: reconcile: could not retrieve historyId");
484 : }
485 :
486 : /* Save label ID→name mapping */
487 : {
488 18 : char **lbl_names = NULL, **lbl_ids = NULL;
489 18 : int lbl_count = 0;
490 18 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
491 18 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
492 180 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
493 18 : free(lbl_names); free(lbl_ids);
494 : }
495 : }
496 :
497 18 : free(all_uids);
498 18 : logger_log(LOG_INFO, "gmail_sync: reconcile done — %d cached, %d queued",
499 : cached, queued);
500 18 : return queued;
501 : }
502 :
503 : /* ── Fetch pending: download queued messages ─────────────────────────── */
504 :
505 : /**
506 : * Download all message UIDs listed in pending_fetch.tsv.
507 : * Removes each entry from the queue on successful download.
508 : * Leaves failures in the queue for retry on the next sync.
509 : *
510 : * Returns number of messages successfully downloaded.
511 : */
512 10 : int gmail_sync_fetch_pending(GmailClient *gc) {
513 10 : int count = 0;
514 10 : char (*uids)[17] = local_pending_fetch_load(&count);
515 10 : if (!uids || count == 0) {
516 0 : free(uids);
517 0 : return 0;
518 : }
519 :
520 10 : logger_log(LOG_INFO, "gmail_sync: fetch_pending — %d messages to download", count);
521 10 : fprintf(stderr, " Downloading %d message(s)...\n", count);
522 :
523 10 : MailRules *rules = mail_rules_load(local_store_account_name());
524 10 : int fetched = 0;
525 : #define PROGRESS_STEP 50
526 610 : for (int i = 0; i < count; i++) {
527 600 : const char *uid = uids[i];
528 :
529 600 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
530 : /* Already present — clean up stale pending entry */
531 1 : local_pending_fetch_remove(uid);
532 1 : continue;
533 : }
534 :
535 599 : if (store_fetched_message(gc, uid, rules) == 0) {
536 599 : local_pending_fetch_remove(uid);
537 599 : fetched++;
538 : }
539 : /* On failure: leave in queue for retry */
540 :
541 599 : if (i % PROGRESS_STEP == 0 || i == count - 1) {
542 26 : fprintf(stderr, "\r\033[K [%d/%d] downloaded", fetched, count);
543 26 : fflush(stderr);
544 : }
545 : }
546 10 : if (count > 0)
547 10 : fprintf(stderr, "\r\033[K %d of %d downloaded\n", fetched, count);
548 :
549 10 : mail_rules_free(rules);
550 10 : free(uids);
551 10 : logger_log(LOG_INFO, "gmail_sync: fetch_pending done — %d/%d downloaded",
552 : fetched, count);
553 10 : return fetched;
554 : }
555 :
556 : /* ── Full Sync ────────────────────────────────────────────────────── */
557 :
558 1 : int gmail_sync_full(GmailClient *gc) {
559 1 : logger_log(LOG_INFO, "gmail_sync: starting full sync");
560 :
561 1 : int queued = gmail_sync_reconcile(gc);
562 1 : if (queued < 0) return -1;
563 :
564 1 : if (queued > 0)
565 0 : gmail_sync_fetch_pending(gc);
566 :
567 : /* Rebuild label indexes from .hdr files so that even when all messages
568 : * were already cached (queued == 0) the indexes are consistent. */
569 : {
570 1 : char (*all_uids)[17] = NULL;
571 1 : int all_count = 0;
572 1 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
573 1 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
574 1 : free(all_uids);
575 : }
576 :
577 1 : return 0;
578 : }
579 :
580 : /* ── History delta processing ─────────────────────────────────────── */
581 :
582 : struct history_ctx {
583 : GmailClient *gc;
584 : MailRules *rules;
585 : int added;
586 : int deleted;
587 : int label_changes;
588 : };
589 :
590 2 : static void process_message_added(const char *obj, int index, void *ctx) {
591 : (void)index;
592 2 : struct history_ctx *hc = ctx;
593 :
594 : /* obj is a history record with "message" sub-object */
595 : /* Extract message ID from the "message" field */
596 : /* The history API returns: {"message": {"id": "...", "labelIds": [...]}} */
597 2 : char *id = json_get_string(obj, "id");
598 2 : if (!id) return;
599 :
600 : /* Fetch and store the new message */
601 2 : char **labels = NULL;
602 2 : int label_count = 0;
603 2 : char *raw = gmail_fetch_message(hc->gc, id, &labels, &label_count);
604 2 : if (raw) {
605 2 : local_msg_save("", id, raw, strlen(raw));
606 :
607 2 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
608 2 : if (hdr) {
609 2 : local_hdr_save("", id, hdr, strlen(hdr));
610 2 : free(hdr);
611 : }
612 :
613 2 : apply_rules_to_new_message(hc->rules, id, raw, labels, label_count);
614 2 : free(raw);
615 :
616 2 : int has_label = 0;
617 6 : for (int j = 0; j < label_count; j++) {
618 4 : if (gmail_sync_is_filtered_label(labels[j])) continue;
619 4 : const char *idx_name = labels[j];
620 4 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
621 4 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
622 4 : label_idx_add(idx_name, id);
623 4 : has_label = 1;
624 : }
625 2 : if (!has_label) {
626 0 : label_idx_add("_nolabel", id);
627 : /* Archived messages are always read */
628 0 : int cur_flags = 0;
629 0 : for (int j = 0; j < label_count; j++) {
630 0 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
631 0 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
632 : }
633 0 : if (cur_flags & MSG_FLAG_UNSEEN)
634 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
635 : }
636 :
637 2 : hc->added++;
638 : }
639 :
640 6 : for (int j = 0; j < label_count; j++) free(labels[j]);
641 2 : free(labels);
642 2 : free(id);
643 : }
644 :
645 0 : static void process_message_deleted(const char *obj, int index, void *ctx) {
646 : (void)index;
647 0 : struct history_ctx *hc = ctx;
648 :
649 0 : char *id = json_get_string(obj, "id");
650 0 : if (!id) return;
651 :
652 0 : local_msg_delete("", id);
653 :
654 : /* Remove from all known label indexes — brute force scan */
655 : /* In practice this is rare; deleted messages are few */
656 0 : char **names = NULL, **ids = NULL;
657 0 : int count = 0;
658 0 : if (gmail_list_labels(hc->gc, &names, &ids, &count) == 0) {
659 0 : for (int i = 0; i < count; i++) {
660 0 : label_idx_remove(ids[i], id);
661 0 : free(names[i]);
662 0 : free(ids[i]);
663 : }
664 0 : free(names);
665 0 : free(ids);
666 : }
667 0 : label_idx_remove("_nolabel", id);
668 0 : label_idx_remove("_spam", id);
669 0 : label_idx_remove("_trash", id);
670 :
671 0 : hc->deleted++;
672 0 : free(id);
673 : }
674 :
675 0 : static void process_labels_added(const char *obj, int index, void *ctx) {
676 : (void)index;
677 0 : struct history_ctx *hc = ctx;
678 :
679 0 : char *id = json_get_string(obj, "id");
680 0 : if (!id) return;
681 :
682 0 : char **add_labels = NULL;
683 0 : int add_count = 0;
684 0 : json_get_string_array(obj, "labelIds", &add_labels, &add_count);
685 :
686 0 : for (int i = 0; i < add_count; i++) {
687 0 : if (gmail_sync_is_filtered_label(add_labels[i])) continue;
688 0 : const char *idx_name = add_labels[i];
689 0 : if (strcmp(add_labels[i], "SPAM") == 0) idx_name = "_spam";
690 0 : else if (strcmp(add_labels[i], "TRASH") == 0) idx_name = "_trash";
691 0 : label_idx_add(idx_name, id);
692 : /* Only remove from _nolabel when a real (non-CATEGORY_) label is added */
693 0 : if (!is_category_label(add_labels[i]))
694 0 : label_idx_remove("_nolabel", id);
695 : }
696 :
697 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate. */
698 0 : local_hdr_update_labels("", id,
699 : (const char **)add_labels, add_count, NULL, 0);
700 :
701 0 : for (int i = 0; i < add_count; i++) free(add_labels[i]);
702 0 : free(add_labels);
703 0 : free(id);
704 0 : hc->label_changes++;
705 : }
706 :
707 0 : static void process_labels_removed(const char *obj, int index, void *ctx) {
708 : (void)index;
709 0 : struct history_ctx *hc = ctx;
710 :
711 0 : char *id = json_get_string(obj, "id");
712 0 : if (!id) return;
713 :
714 0 : char **rm_labels = NULL;
715 0 : int rm_count = 0;
716 0 : json_get_string_array(obj, "labelIds", &rm_labels, &rm_count);
717 :
718 0 : for (int i = 0; i < rm_count; i++) {
719 0 : if (gmail_sync_is_filtered_label(rm_labels[i])) continue;
720 0 : const char *idx_name = rm_labels[i];
721 0 : if (strcmp(rm_labels[i], "SPAM") == 0) idx_name = "_spam";
722 0 : else if (strcmp(rm_labels[i], "TRASH") == 0) idx_name = "_trash";
723 0 : label_idx_remove(idx_name, id);
724 : }
725 :
726 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate.
727 : * Must be done BEFORE freeing rm_labels (used-after-free guard). */
728 0 : local_hdr_update_labels("", id,
729 : NULL, 0, (const char **)rm_labels, rm_count);
730 :
731 0 : for (int i = 0; i < rm_count; i++) free(rm_labels[i]);
732 0 : free(rm_labels);
733 :
734 : /* Check if any labels remain; if none → add to _nolabel */
735 : /* Quick check: fetch message labels from server */
736 0 : char **cur_labels = NULL;
737 0 : int cur_count = 0;
738 0 : char *raw = gmail_fetch_message(hc->gc, id, &cur_labels, &cur_count);
739 0 : free(raw);
740 :
741 0 : int has_real_label = 0;
742 0 : for (int i = 0; i < cur_count; i++) {
743 0 : if (!gmail_sync_is_filtered_label(cur_labels[i]) &&
744 0 : !is_category_label(cur_labels[i]))
745 0 : has_real_label = 1;
746 0 : free(cur_labels[i]);
747 : }
748 0 : free(cur_labels);
749 :
750 0 : if (!has_real_label) {
751 0 : label_idx_add("_nolabel", id);
752 : /* Archived messages are always read: clear UNSEEN from .hdr flags */
753 0 : char *cur_hdr = local_hdr_load("", id);
754 0 : if (cur_hdr) {
755 0 : char *last_tab = strrchr(cur_hdr, '\t');
756 0 : if (last_tab) {
757 0 : int cur_flags = atoi(last_tab + 1);
758 0 : if (cur_flags & MSG_FLAG_UNSEEN)
759 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
760 : }
761 0 : free(cur_hdr);
762 : }
763 : }
764 :
765 0 : free(id);
766 0 : hc->label_changes++;
767 : }
768 :
769 : /* ── One-time repair: archived messages must not be unread ─────────── */
770 :
771 4 : void gmail_sync_repair_archive_flags(void) {
772 4 : char (*uids)[17] = NULL;
773 4 : int count = 0;
774 4 : if (label_idx_load("_nolabel", &uids, &count) != 0 || count == 0) {
775 4 : free(uids);
776 4 : return;
777 : }
778 0 : for (int i = 0; i < count; i++) {
779 0 : char *hdr = local_hdr_load("", uids[i]);
780 0 : if (!hdr) continue;
781 0 : char *last_tab = strrchr(hdr, '\t');
782 0 : if (last_tab) {
783 0 : int flags = atoi(last_tab + 1);
784 0 : if (flags & MSG_FLAG_UNSEEN)
785 0 : local_hdr_update_flags("", uids[i], flags & ~MSG_FLAG_UNSEEN);
786 : }
787 0 : free(hdr);
788 : }
789 0 : free(uids);
790 : }
791 :
792 : /* ── Incremental Sync ─────────────────────────────────────────────── */
793 :
794 6 : int gmail_sync_incremental(GmailClient *gc) {
795 6 : char *history_id = local_gmail_history_load();
796 6 : if (!history_id) {
797 0 : logger_log(LOG_INFO, "gmail_sync: no historyId, need full sync");
798 0 : return -2;
799 : }
800 :
801 6 : logger_log(LOG_INFO, "gmail_sync: incremental from historyId %s", history_id);
802 :
803 6 : char *resp = gmail_get_history(gc, history_id);
804 6 : free(history_id);
805 :
806 6 : if (!resp) {
807 2 : fprintf(stderr, " Incremental: History API returned error/404 (historyId expired or network issue).\n");
808 2 : logger_log(LOG_WARN, "gmail_sync: history expired or error");
809 2 : return -2; /* Signal: need full sync */
810 : }
811 :
812 4 : MailRules *inc_rules = mail_rules_load(local_store_account_name());
813 4 : struct history_ctx hc = { .gc = gc, .rules = inc_rules, .added = 0, .deleted = 0, .label_changes = 0 };
814 :
815 : /* Process each history record */
816 : /* The history response has: {"history": [{...}, ...], "historyId": "..."} */
817 : /* Each history entry may contain messagesAdded, messagesDeleted,
818 : * labelsAdded, labelsRemoved arrays */
819 :
820 : /* Process delta events from the history response.
821 : * Gmail nests events inside history records, but our json_foreach_object
822 : * searches the full response for matching keys, so a single scan works. */
823 4 : json_foreach_object(resp, "messagesAdded", process_message_added, &hc);
824 4 : json_foreach_object(resp, "messagesDeleted", process_message_deleted, &hc);
825 4 : json_foreach_object(resp, "labelsAdded", process_labels_added, &hc);
826 4 : json_foreach_object(resp, "labelsRemoved", process_labels_removed, &hc);
827 :
828 : /* Save updated historyId */
829 4 : RAII_STRING char *new_history_id = json_get_string(resp, "historyId");
830 4 : if (new_history_id)
831 4 : local_gmail_history_save(new_history_id);
832 :
833 4 : free(resp);
834 :
835 : /* Refresh label name mapping if any label events occurred */
836 4 : if (hc.label_changes > 0) {
837 0 : char **lbl_names = NULL, **lbl_ids = NULL;
838 0 : int lbl_count = 0;
839 0 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
840 0 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
841 0 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
842 0 : free(lbl_names);
843 0 : free(lbl_ids);
844 : }
845 : }
846 :
847 : /* Ensure no archived message is marked unread (repair existing data too) */
848 4 : gmail_sync_repair_archive_flags();
849 :
850 4 : mail_rules_free(inc_rules);
851 4 : logger_log(LOG_INFO, "gmail_sync: incremental done — added=%d deleted=%d labels=%d",
852 : hc.added, hc.deleted, hc.label_changes);
853 4 : return 0;
854 : }
855 :
856 : /* ── Auto Sync (public entry point) ───────────────────────────────── */
857 :
858 : /**
859 : * Smart sync flow:
860 : *
861 : * 1. If pending_fetch.tsv is non-empty, download those first (resuming an
862 : * interrupted previous sync or initial download).
863 : * 2. If the local store was already complete (no pending at start) AND a
864 : * valid historyId exists, use the fast incremental path (1–2 API calls).
865 : * 3. Otherwise, run reconcile (full UID listing) to discover any missing
866 : * messages, then download them.
867 : *
868 : * This ensures:
869 : * - First run or expired historyId: O(N) reconcile → O(missing) downloads.
870 : * - Subsequent runs on a mature store: O(1) incremental (no listing at all).
871 : * - Interrupted downloads: resume from pending_fetch.tsv without re-listing.
872 : */
873 21 : int gmail_sync(GmailClient *gc) {
874 : /* Step 1: check readiness before downloading anything */
875 21 : int had_pending = local_pending_fetch_count() > 0;
876 :
877 : /* Step 2: drain any queued downloads from a previous (possibly interrupted) sync */
878 21 : if (had_pending)
879 1 : gmail_sync_fetch_pending(gc);
880 :
881 : /* Step 3: try fast incremental path if we have a saved historyId.
882 : * We do this regardless of whether there were pending downloads —
883 : * draining pending_fetch.tsv already brought the local store up to the
884 : * reconcile snapshot; incremental then catches anything that arrived
885 : * on the server after that snapshot. */
886 21 : char *history_id = local_gmail_history_load();
887 21 : int have_history = (history_id != NULL);
888 21 : free(history_id);
889 :
890 21 : if (have_history) {
891 6 : fprintf(stderr, " Incremental sync (historyId present)...\n");
892 6 : int rc = gmail_sync_incremental(gc);
893 6 : if (rc == 0) {
894 4 : fprintf(stderr, " Incremental sync: up to date.\n");
895 4 : return 0; /* fast path — done */
896 : }
897 2 : if (rc != -2) return rc; /* unexpected error */
898 2 : fprintf(stderr, " Incremental sync: historyId expired — falling back to full reconcile.\n");
899 2 : logger_log(LOG_INFO, "gmail_sync: historyId expired, falling back to reconcile");
900 : } else {
901 15 : fprintf(stderr, " No saved historyId — full reconcile needed.\n");
902 : }
903 :
904 : /* Step 4: reconcile (discover what is missing) */
905 17 : int queued = gmail_sync_reconcile(gc);
906 17 : if (queued < 0) return -1;
907 :
908 : /* Step 5: download what reconcile found */
909 17 : if (queued > 0)
910 9 : gmail_sync_fetch_pending(gc);
911 :
912 : /* Step 6: rebuild label indexes from .hdr files.
913 : * Necessary when all messages were already cached (queued == 0) but
914 : * label .idx files were deleted or are missing (e.g. manual deletion
915 : * or upgrade from an older version). */
916 : {
917 17 : char (*all_uids)[17] = NULL;
918 17 : int all_count = 0;
919 17 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
920 17 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
921 17 : free(all_uids);
922 : }
923 :
924 17 : return 0;
925 : }
|
