Line data Source code
1 : #include "gmail_sync.h"
2 : #include "gmail_client.h"
3 : #include "local_store.h"
4 : #include "mail_rules.h"
5 : #include "mime_util.h"
6 : #include "json_util.h"
7 : #include "logger.h"
8 : #include "raii.h"
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <string.h>
12 : #include <stddef.h>
13 :
14 : /* ── Progress callbacks ───────────────────────────────────────────── */
15 :
16 : /* Called by gmail_list_messages after each page: cur = messages collected so far */
17 71 : static void list_progress_cb(size_t cur, size_t total, void *ctx) {
18 : (void)total; (void)ctx;
19 71 : fprintf(stderr, "\r\033[K Listing messages... %zu found", cur);
20 71 : fflush(stderr);
21 71 : }
22 :
23 : /* ── Gmail .hdr file format ───────────────────────────────────────── */
24 :
25 : /**
26 : * Build a .hdr string from raw message headers and label list.
27 : * Format: from\tsubject\tdate\tlabel1,label2,...\tflags\n
28 : *
29 : * Returns heap-allocated string. Caller must free().
30 : */
31 816 : char *gmail_sync_build_hdr(const char *raw_msg, char **labels, int label_count) {
32 1632 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
33 1632 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
34 1632 : RAII_STRING char *date_raw = mime_get_header(raw_msg, "Date");
35 :
36 1632 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
37 1632 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
38 1632 : RAII_STRING char *date_fmt = date_raw ? mime_format_date(date_raw) : NULL;
39 :
40 816 : const char *from = from_dec ? from_dec : "";
41 816 : const char *subj = subj_dec ? subj_dec : "";
42 816 : const char *date = date_fmt ? date_fmt : "";
43 :
44 : /* Build comma-separated label string */
45 816 : size_t lbl_len = 1;
46 2218 : for (int i = 0; i < label_count; i++)
47 1402 : lbl_len += strlen(labels[i]) + 1;
48 816 : char *lbl_str = calloc(lbl_len, 1);
49 816 : if (lbl_str) {
50 2218 : for (int i = 0; i < label_count; i++) {
51 1402 : if (i > 0) strcat(lbl_str, ",");
52 1402 : strcat(lbl_str, labels[i]);
53 : }
54 : }
55 :
56 : /* Compute flags bitmask from labels */
57 816 : int flags = 0;
58 2218 : for (int i = 0; i < label_count; i++) {
59 1402 : if (strcmp(labels[i], "UNREAD") == 0) flags |= MSG_FLAG_UNSEEN;
60 1402 : if (strcmp(labels[i], "STARRED") == 0) flags |= MSG_FLAG_FLAGGED;
61 1402 : if (strcmp(labels[i], "SPAM") == 0) flags |= MSG_FLAG_JUNK;
62 : }
63 :
64 : /* Compute attachment and DMARC flags from the raw message so the TUI
65 : * never needs to download anything to display these indicators. */
66 1632 : RAII_STRING char *ct_raw = mime_get_header(raw_msg, "Content-Type");
67 816 : if (ct_raw && strcasestr(ct_raw, "multipart/mixed"))
68 58 : flags |= MSG_FLAG_ATTACH;
69 816 : flags |= MSG_FLAG_ATTACH_CHECKED;
70 :
71 816 : RAII_STRING char *ar_raw = mime_get_header(raw_msg, "Authentication-Results");
72 816 : int dmarc_st = mime_get_dmarc_status(ar_raw);
73 816 : if (dmarc_st == 1) flags |= MSG_FLAG_DMARC_PASS;
74 816 : else if (dmarc_st == -1) flags |= MSG_FLAG_DMARC_FAIL;
75 816 : if (dmarc_st != -2) flags |= MSG_FLAG_DMARC_CHECKED;
76 :
77 : /* Replace tabs in fields with spaces */
78 816 : char *hdr = NULL;
79 816 : if (asprintf(&hdr, "%s\t%s\t%s\t%s\t%d",
80 : from, subj, date, lbl_str ? lbl_str : "", flags) == -1)
81 0 : hdr = NULL;
82 816 : free(lbl_str);
83 :
84 : /* Sanitise: replace any tabs within field values */
85 816 : if (hdr) {
86 : /* The first 4 tabs are field separators; tabs within values got
87 : * inserted by asprintf if field values contained tabs. Since we
88 : * used tab as separator this is inherently safe (mime_decode_words
89 : * doesn't produce tabs), but defend anyway. */
90 : }
91 :
92 816 : return hdr;
93 : }
94 :
95 : /* ── Filtered labels (metadata-only, excluded from indexing) ──────── */
96 :
97 7427 : int gmail_sync_is_filtered_label(const char *label_id) {
98 7427 : if (!label_id) return 1;
99 7427 : if (strcmp(label_id, "IMPORTANT") == 0) return 1;
100 7427 : if (strcmp(label_id, "CHAT") == 0) return 1;
101 7427 : return 0;
102 : }
103 :
104 : /* Returns 1 if label_id is a Gmail automatic inbox category (CATEGORY_*).
105 : * Category labels are indexed like user labels, but a message whose ONLY
106 : * non-filtered labels are CATEGORY_* is also added to _nolabel (Archive). */
107 7423 : static int is_category_label(const char *label_id) {
108 7423 : return label_id && strncmp(label_id, "CATEGORY_", 9) == 0;
109 : }
110 :
111 : /* ── Mail rules helper ───────────────────────────────────────────── */
112 :
113 : /* Apply mail rules to a newly stored message.
114 : * Builds labels_csv from Gmail label IDs (resolving user labels to names),
115 : * calls mail_rules_apply(), then updates the local .hdr and label indexes. */
116 816 : static void apply_rules_to_new_message(const MailRules *rules, const char *uid,
117 : const char *raw_msg,
118 : char **labels, int label_count)
119 : {
120 816 : if (!rules || rules->count == 0) return;
121 :
122 0 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
123 0 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
124 0 : RAII_STRING char *to_raw = mime_get_header(raw_msg, "To");
125 0 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
126 0 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
127 0 : RAII_STRING char *to_dec = to_raw ? mime_decode_words(to_raw) : NULL;
128 :
129 : /* Build labels_csv using friendly names where available */
130 0 : size_t lcsz = 1;
131 0 : for (int i = 0; i < label_count; i++) {
132 0 : char *name = local_gmail_label_name_lookup(labels[i]);
133 0 : lcsz += strlen(name ? name : labels[i]) + 2;
134 0 : free(name);
135 : }
136 0 : char *lcsv = calloc(lcsz, 1);
137 0 : if (!lcsv) return;
138 0 : for (int i = 0; i < label_count; i++) {
139 0 : char *name = local_gmail_label_name_lookup(labels[i]);
140 0 : const char *display = name ? name : labels[i];
141 0 : if (lcsv[0]) strcat(lcsv, ",");
142 0 : strcat(lcsv, display);
143 0 : free(name);
144 : }
145 :
146 0 : char **add_out = NULL; int add_count = 0;
147 0 : char **rm_out = NULL; int rm_count = 0;
148 0 : int fired = mail_rules_apply(rules,
149 : from_dec, subj_dec, to_dec, lcsv,
150 : NULL, (time_t)0, /* body/date unavailable during Gmail sync */
151 : &add_out, &add_count,
152 : &rm_out, &rm_count);
153 0 : free(lcsv);
154 0 : if (fired <= 0) return;
155 :
156 0 : logger_log(LOG_INFO, "gmail_sync: rules fired=%d for %s (add=%d rm=%d)",
157 : fired, uid, add_count, rm_count);
158 :
159 : /* Update local .hdr and label indexes */
160 0 : local_hdr_update_labels("", uid,
161 : (const char **)add_out, add_count,
162 : (const char **)rm_out, rm_count);
163 0 : for (int i = 0; i < add_count; i++) {
164 0 : label_idx_add(add_out[i], uid);
165 0 : free(add_out[i]);
166 : }
167 0 : for (int i = 0; i < rm_count; i++) {
168 0 : label_idx_remove(rm_out[i], uid);
169 0 : free(rm_out[i]);
170 : }
171 0 : free(add_out);
172 0 : free(rm_out);
173 :
174 : /* Update contact suggestion cache */
175 : {
176 0 : char *from_h = mime_get_header(raw_msg, "From");
177 0 : char *to_h = mime_get_header(raw_msg, "To");
178 0 : char *cc_h = mime_get_header(raw_msg, "Cc");
179 0 : local_contacts_update(from_h, to_h, cc_h);
180 0 : free(from_h); free(to_h); free(cc_h);
181 : }
182 : }
183 :
184 : /* ── Label index rebuild ──────────────────────────────────────────── */
185 :
186 : typedef struct { char label[64]; char uid[17]; } LabelUidPair;
187 :
188 38973 : static int cmp_lbl_uid_pair(const void *a, const void *b) {
189 38973 : const LabelUidPair *pa = a, *pb = b;
190 38973 : int c = strcmp(pa->label, pb->label);
191 38973 : return c ? c : strcmp(pa->uid, pb->uid);
192 : }
193 :
194 : /**
195 : * Rebuild ALL label .idx files from the .hdr files for the given UIDs.
196 : *
197 : * Efficient O(N log N) approach:
198 : * 1. Read each .hdr and collect (label, uid) pairs in memory.
199 : * 2. Sort the flat pair array.
200 : * 3. Write each label's .idx file in one grouped pass.
201 : *
202 : * This is called at the end of every full sync so that cached messages
203 : * (whose .idx entries were never written) are correctly indexed.
204 : */
205 71 : static void rebuild_label_indexes(const char (*uids)[17], int uid_count) {
206 71 : if (uid_count <= 0) return;
207 :
208 71 : fprintf(stderr, " Rebuilding label indexes...");
209 71 : fflush(stderr);
210 :
211 : /* Phase 1: collect (label, uid) pairs from all .hdr files */
212 71 : size_t cap = (size_t)uid_count * 5; /* ~5 labels per message */
213 71 : LabelUidPair *pairs = malloc(cap * sizeof(LabelUidPair));
214 71 : if (!pairs) {
215 0 : fprintf(stderr, " [out of memory]\n");
216 0 : return;
217 : }
218 71 : int npairs = 0;
219 :
220 3859 : for (int i = 0; i < uid_count; i++) {
221 : /* Load full .hdr so we can both collect label pairs and sync the
222 : * flags integer in one read (avoid a separate local_hdr_get_labels
223 : * call followed by local_hdr_update_labels). */
224 3788 : char *hdr = local_hdr_load("", uids[i]);
225 3788 : if (!hdr) continue;
226 :
227 : /* Locate labels field (4th tab-separated token).
228 : * Track the tab pointer so we can NUL-terminate the prefix later. */
229 3788 : char *t3_tab = hdr;
230 15152 : for (int f = 0; f < 3; f++) {
231 11364 : t3_tab = strchr(t3_tab, '\t');
232 11364 : if (!t3_tab) break;
233 11364 : if (f < 2) t3_tab++;
234 : }
235 3788 : if (!t3_tab || t3_tab == hdr) { free(hdr); continue; }
236 3788 : char *lbl_start = t3_tab + 1; /* start of labels CSV field */
237 :
238 : /* Locate optional flags field (5th token) and read old value */
239 3788 : char *t4 = strchr(lbl_start, '\t');
240 3788 : int old_flags = 0;
241 3788 : if (t4) {
242 3788 : old_flags = atoi(t4 + 1);
243 3788 : *t4 = '\0'; /* NUL-terminate labels field in-place */
244 : } else {
245 0 : char *nl = strchr(lbl_start, '\n');
246 0 : if (nl) *nl = '\0';
247 : }
248 :
249 : /* Derive new flags from labels (preserving non-label bits) */
250 3788 : int new_flags = old_flags & ~(MSG_FLAG_UNSEEN | MSG_FLAG_FLAGGED);
251 3788 : int has_real = 0;
252 :
253 : /* Iterate labels via a copy (tokenising modifies the string) */
254 3788 : char *lbl_copy = strdup(lbl_start);
255 3788 : if (!lbl_copy) { free(hdr); continue; }
256 :
257 3788 : char *tok = lbl_copy;
258 9816 : while (tok) {
259 6028 : char *comma = strchr(tok, ',');
260 6028 : if (comma) *comma = '\0';
261 :
262 6028 : if (tok[0] && !gmail_sync_is_filtered_label(tok)) {
263 6025 : const char *idx_name = tok;
264 6025 : if (strcmp(tok, "SPAM") == 0) idx_name = "_spam";
265 6025 : else if (strcmp(tok, "TRASH") == 0) idx_name = "_trash";
266 :
267 6025 : if (npairs >= (int)cap) {
268 0 : cap = cap * 2 + 1;
269 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
270 0 : if (!tmp) { free(lbl_copy); free(hdr); free(pairs); return; }
271 0 : pairs = tmp;
272 : }
273 6025 : strncpy(pairs[npairs].label, idx_name, 63);
274 6025 : pairs[npairs].label[63] = '\0';
275 6025 : strncpy(pairs[npairs].uid, uids[i], 16);
276 6025 : pairs[npairs].uid[16] = '\0';
277 6025 : npairs++;
278 :
279 6025 : if (!is_category_label(tok)) has_real = 1;
280 6025 : if (strcmp(tok, "UNREAD") == 0) new_flags |= MSG_FLAG_UNSEEN;
281 6025 : if (strcmp(tok, "STARRED") == 0) new_flags |= MSG_FLAG_FLAGGED;
282 : }
283 6028 : tok = comma ? comma + 1 : NULL;
284 : }
285 3788 : free(lbl_copy);
286 :
287 : /* Messages with no real (non-CATEGORY_) label → Archive.
288 : * Archived messages are always considered read. */
289 3788 : if (!has_real) {
290 3 : if (npairs >= (int)cap) {
291 0 : cap = cap * 2 + 1;
292 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
293 0 : if (!tmp) { free(hdr); free(pairs); return; }
294 0 : pairs = tmp;
295 : }
296 3 : strncpy(pairs[npairs].label, "_nolabel", 63);
297 3 : strncpy(pairs[npairs].uid, uids[i], 16);
298 3 : pairs[npairs].uid[16] = '\0';
299 3 : npairs++;
300 3 : new_flags &= ~MSG_FLAG_UNSEEN;
301 : }
302 :
303 : /* Sync flags integer if it disagrees with the labels CSV.
304 : * lbl_start still points into hdr at the labels field.
305 : * NUL-terminate the prefix at t3_tab then reassemble. */
306 3788 : if (new_flags != old_flags) {
307 0 : *t3_tab = '\0';
308 0 : char *updated = NULL;
309 0 : if (asprintf(&updated, "%s\t%s\t%d", hdr, lbl_start, new_flags) != -1) {
310 0 : local_hdr_save("", uids[i], updated, strlen(updated));
311 0 : free(updated);
312 : }
313 : }
314 :
315 3788 : free(hdr);
316 : }
317 :
318 : /* Phase 2: sort by (label, uid) */
319 71 : qsort(pairs, (size_t)npairs, sizeof(LabelUidPair), cmp_lbl_uid_pair);
320 :
321 : /* Phase 3: group by label and write each .idx file */
322 71 : int labels_written = 0;
323 71 : int i = 0;
324 268 : while (i < npairs) {
325 197 : const char *cur_label = pairs[i].label;
326 197 : int j = i;
327 6225 : while (j < npairs && strcmp(pairs[j].label, cur_label) == 0) j++;
328 197 : int run = j - i;
329 :
330 197 : char (*uid_arr)[17] = malloc((size_t)run * sizeof(char[17]));
331 197 : if (uid_arr) {
332 197 : int unique = 0;
333 6225 : for (int k = i; k < j; k++) {
334 6028 : if (unique == 0 ||
335 5831 : strcmp(uid_arr[unique - 1], pairs[k].uid) != 0) {
336 6028 : memcpy(uid_arr[unique++], pairs[k].uid, 17);
337 : }
338 : }
339 197 : label_idx_write(cur_label, (const char (*)[17])uid_arr, unique);
340 197 : free(uid_arr);
341 197 : labels_written++;
342 : }
343 197 : i = j;
344 : }
345 71 : free(pairs);
346 :
347 71 : fprintf(stderr, "\r\033[K Label indexes rebuilt (%d labels)\n",
348 : labels_written);
349 71 : logger_log(LOG_INFO,
350 : "gmail_sync: rebuilt %d label indexes from %d messages",
351 : labels_written, uid_count);
352 : }
353 :
354 : /**
355 : * Rebuild all label .idx files from locally cached .hdr files.
356 : * Does NOT contact the Gmail API.
357 : * Use this to repair missing or incomplete indexes without re-downloading.
358 : */
359 9 : int gmail_sync_rebuild_indexes(void) {
360 9 : char (*uids)[17] = NULL;
361 9 : int count = 0;
362 9 : if (local_hdr_list_all_uids("", &uids, &count) != 0) {
363 0 : fprintf(stderr, "Error: could not scan local message store.\n");
364 0 : return -1;
365 : }
366 9 : fprintf(stderr, " Found %d cached messages.\n", count);
367 9 : rebuild_label_indexes((const char (*)[17])uids, count);
368 9 : free(uids);
369 9 : return 0;
370 : }
371 :
372 : /* ── Single message fetch+store helper ───────────────────────────────── */
373 :
374 : /**
375 : * Fetch one message from the Gmail API, save .eml + .hdr, apply rules,
376 : * update label indexes.
377 : * Returns 0 on success, -1 on fetch error (transient; caller should retry).
378 : */
379 814 : static int store_fetched_message(GmailClient *gc, const char *uid,
380 : const MailRules *rules)
381 : {
382 814 : char **labels = NULL;
383 814 : int label_count = 0;
384 814 : char *raw = gmail_fetch_message(gc, uid, &labels, &label_count);
385 814 : if (!raw) {
386 0 : logger_log(LOG_WARN, "gmail_sync: failed to fetch %s", uid);
387 0 : for (int j = 0; j < label_count; j++) free(labels[j]);
388 0 : free(labels);
389 0 : return -1;
390 : }
391 :
392 814 : local_msg_save("", uid, raw, strlen(raw));
393 :
394 814 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
395 814 : if (hdr) { local_hdr_save("", uid, hdr, strlen(hdr)); free(hdr); }
396 :
397 814 : apply_rules_to_new_message(rules, uid, raw, labels, label_count);
398 814 : free(raw);
399 :
400 814 : int has_real_label = 0;
401 2212 : for (int j = 0; j < label_count; j++) {
402 1398 : if (gmail_sync_is_filtered_label(labels[j])) continue;
403 1398 : const char *idx_name = labels[j];
404 1398 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
405 1398 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
406 1398 : label_idx_add(idx_name, uid);
407 1398 : if (!is_category_label(labels[j])) has_real_label = 1;
408 : }
409 814 : if (!has_real_label) {
410 0 : label_idx_add("_nolabel", uid);
411 0 : int cur_flags = 0;
412 0 : for (int j = 0; j < label_count; j++) {
413 0 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
414 0 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
415 : }
416 0 : if (cur_flags & MSG_FLAG_UNSEEN)
417 0 : local_hdr_update_flags("", uid, cur_flags & ~MSG_FLAG_UNSEEN);
418 : }
419 :
420 2212 : for (int j = 0; j < label_count; j++) free(labels[j]);
421 814 : free(labels);
422 814 : return 0;
423 : }
424 :
425 : /* ── Reconcile: discover missing UIDs and queue them ─────────────────── */
426 :
427 : /**
428 : * List all server-side message IDs, compare with the local store, and
429 : * add any missing UIDs to pending_fetch.tsv. Does NOT download messages.
430 : *
431 : * Also updates the historyId and label name mapping so subsequent
432 : * incremental syncs know where to resume from.
433 : *
434 : * Returns the number of UIDs added to the pending-fetch queue, or -1 on
435 : * a fatal error (e.g. the server cannot be reached).
436 : */
437 62 : int gmail_sync_reconcile(GmailClient *gc) {
438 62 : logger_log(LOG_INFO, "gmail_sync: reconcile — listing server messages");
439 :
440 62 : fprintf(stderr, " Listing messages...");
441 62 : fflush(stderr);
442 62 : gmail_set_progress(gc, list_progress_cb, NULL);
443 :
444 62 : char (*all_uids)[17] = NULL;
445 62 : int uid_count = 0;
446 62 : char *list_history_id = NULL;
447 62 : if (gmail_list_messages(gc, NULL, NULL, &all_uids, &uid_count, &list_history_id) != 0) {
448 0 : gmail_set_progress(gc, NULL, NULL);
449 0 : free(list_history_id);
450 0 : logger_log(LOG_ERROR, "gmail_sync: reconcile failed to list messages");
451 0 : return -1;
452 : }
453 62 : gmail_set_progress(gc, NULL, NULL);
454 62 : fprintf(stderr, "\r\033[K %d messages on server\n", uid_count);
455 :
456 : /* Clear any stale pending_fetch entries before repopulating */
457 62 : local_pending_fetch_clear();
458 :
459 62 : int queued = 0, cached = 0;
460 2100 : for (int i = 0; i < uid_count; i++) {
461 2038 : const char *uid = all_uids[i];
462 2038 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
463 1224 : cached++;
464 1224 : if (i % 500 == 0 || i == uid_count - 1) {
465 20 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
466 : i + 1, uid_count);
467 20 : fflush(stderr);
468 : }
469 1224 : continue;
470 : }
471 814 : local_pending_fetch_add(uid);
472 814 : queued++;
473 814 : if ((cached + queued) % 500 == 0 || i == uid_count - 1) {
474 52 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
475 : i + 1, uid_count);
476 52 : fflush(stderr);
477 : }
478 : }
479 62 : if (uid_count > 0)
480 62 : fprintf(stderr, "\r\033[K %d cached, %d queued for download\n",
481 : cached, queued);
482 :
483 : /* Save historyId so next run can use incremental sync.
484 : * Prefer the historyId from the messages.list response (always fresh);
485 : * fall back to the /profile endpoint only if that field was absent. */
486 62 : if (list_history_id) {
487 62 : fprintf(stderr, " historyId from list response: %s\n", list_history_id);
488 62 : local_gmail_history_save(list_history_id);
489 62 : free(list_history_id);
490 62 : list_history_id = NULL;
491 : } else {
492 0 : RAII_STRING char *hid = gmail_get_history_id(gc);
493 0 : if (hid)
494 0 : local_gmail_history_save(hid);
495 : else
496 0 : logger_log(LOG_WARN, "gmail_sync: reconcile: could not retrieve historyId");
497 : }
498 :
499 : /* Save label ID→name mapping */
500 : {
501 62 : char **lbl_names = NULL, **lbl_ids = NULL;
502 62 : int lbl_count = 0;
503 62 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
504 62 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
505 620 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
506 62 : free(lbl_names); free(lbl_ids);
507 : }
508 : }
509 :
510 62 : free(all_uids);
511 62 : logger_log(LOG_INFO, "gmail_sync: reconcile done — %d cached, %d queued",
512 : cached, queued);
513 62 : return queued;
514 : }
515 :
516 : /* ── Fetch pending: download queued messages ─────────────────────────── */
517 :
518 : /**
519 : * Download all message UIDs listed in pending_fetch.tsv.
520 : * Removes each entry from the queue on successful download.
521 : * Leaves failures in the queue for retry on the next sync.
522 : *
523 : * Returns number of messages successfully downloaded.
524 : */
525 55 : int gmail_sync_fetch_pending(GmailClient *gc) {
526 55 : int count = 0;
527 55 : char (*uids)[17] = local_pending_fetch_load(&count);
528 55 : if (!uids || count == 0) {
529 0 : free(uids);
530 0 : return 0;
531 : }
532 :
533 55 : logger_log(LOG_INFO, "gmail_sync: fetch_pending — %d messages to download", count);
534 55 : fprintf(stderr, " Downloading %d message(s)...\n", count);
535 :
536 55 : MailRules *rules = mail_rules_load(local_store_account_name());
537 55 : int fetched = 0;
538 : #define PROGRESS_STEP 50
539 872 : for (int i = 0; i < count; i++) {
540 817 : const char *uid = uids[i];
541 :
542 817 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
543 : /* Already present — clean up stale pending entry */
544 3 : local_pending_fetch_remove(uid);
545 3 : continue;
546 : }
547 :
548 814 : if (store_fetched_message(gc, uid, rules) == 0) {
549 814 : local_pending_fetch_remove(uid);
550 814 : fetched++;
551 : }
552 : /* On failure: leave in queue for retry */
553 :
554 814 : if (i % PROGRESS_STEP == 0 || i == count - 1) {
555 112 : fprintf(stderr, "\r\033[K [%d/%d] downloaded", fetched, count);
556 112 : fflush(stderr);
557 : }
558 : }
559 55 : if (count > 0)
560 55 : fprintf(stderr, "\r\033[K %d of %d downloaded\n", fetched, count);
561 :
562 55 : mail_rules_free(rules);
563 55 : free(uids);
564 55 : logger_log(LOG_INFO, "gmail_sync: fetch_pending done — %d/%d downloaded",
565 : fetched, count);
566 55 : return fetched;
567 : }
568 :
569 : /* ── Full Sync ────────────────────────────────────────────────────── */
570 :
571 1 : int gmail_sync_full(GmailClient *gc) {
572 1 : logger_log(LOG_INFO, "gmail_sync: starting full sync");
573 :
574 1 : int queued = gmail_sync_reconcile(gc);
575 1 : if (queued < 0) return -1;
576 :
577 1 : if (queued > 0)
578 0 : gmail_sync_fetch_pending(gc);
579 :
580 : /* Rebuild label indexes from .hdr files so that even when all messages
581 : * were already cached (queued == 0) the indexes are consistent. */
582 : {
583 1 : char (*all_uids)[17] = NULL;
584 1 : int all_count = 0;
585 1 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
586 1 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
587 1 : free(all_uids);
588 : }
589 :
590 1 : return 0;
591 : }
592 :
593 : /* ── History delta processing ─────────────────────────────────────── */
594 :
595 : struct history_ctx {
596 : GmailClient *gc;
597 : MailRules *rules;
598 : int added;
599 : int deleted;
600 : int label_changes;
601 : };
602 :
603 2 : static void process_message_added(const char *obj, int index, void *ctx) {
604 : (void)index;
605 2 : struct history_ctx *hc = ctx;
606 :
607 : /* Gmail history item: {"message": {"id": "...", "labelIds": [...]}} */
608 2 : char *id = json_get_nested_string(obj, "message", "id");
609 2 : if (!id) return;
610 :
611 : /* Fetch and store the new message */
612 2 : char **labels = NULL;
613 2 : int label_count = 0;
614 2 : char *raw = gmail_fetch_message(hc->gc, id, &labels, &label_count);
615 2 : if (raw) {
616 2 : local_msg_save("", id, raw, strlen(raw));
617 :
618 2 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
619 2 : if (hdr) {
620 2 : local_hdr_save("", id, hdr, strlen(hdr));
621 2 : free(hdr);
622 : }
623 :
624 2 : apply_rules_to_new_message(hc->rules, id, raw, labels, label_count);
625 2 : free(raw);
626 :
627 2 : int has_label = 0;
628 6 : for (int j = 0; j < label_count; j++) {
629 4 : if (gmail_sync_is_filtered_label(labels[j])) continue;
630 4 : const char *idx_name = labels[j];
631 4 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
632 4 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
633 4 : label_idx_add(idx_name, id);
634 4 : has_label = 1;
635 : }
636 2 : if (!has_label) {
637 0 : label_idx_add("_nolabel", id);
638 : /* Archived messages are always read */
639 0 : int cur_flags = 0;
640 0 : for (int j = 0; j < label_count; j++) {
641 0 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
642 0 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
643 : }
644 0 : if (cur_flags & MSG_FLAG_UNSEEN)
645 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
646 : }
647 :
648 2 : hc->added++;
649 : }
650 :
651 6 : for (int j = 0; j < label_count; j++) free(labels[j]);
652 2 : free(labels);
653 2 : free(id);
654 : }
655 :
656 0 : static void process_message_deleted(const char *obj, int index, void *ctx) {
657 : (void)index;
658 0 : struct history_ctx *hc = ctx;
659 :
660 : /* Gmail history item: {"message": {"id": "..."}} */
661 0 : char *id = json_get_nested_string(obj, "message", "id");
662 0 : if (!id) return;
663 :
664 0 : local_msg_delete("", id);
665 :
666 : /* Remove from all known label indexes — brute force scan */
667 : /* In practice this is rare; deleted messages are few */
668 0 : char **names = NULL, **ids = NULL;
669 0 : int count = 0;
670 0 : if (gmail_list_labels(hc->gc, &names, &ids, &count) == 0) {
671 0 : for (int i = 0; i < count; i++) {
672 0 : label_idx_remove(ids[i], id);
673 0 : free(names[i]);
674 0 : free(ids[i]);
675 : }
676 0 : free(names);
677 0 : free(ids);
678 : }
679 0 : label_idx_remove("_nolabel", id);
680 0 : label_idx_remove("_spam", id);
681 0 : label_idx_remove("_trash", id);
682 :
683 0 : hc->deleted++;
684 0 : free(id);
685 : }
686 :
687 0 : static void process_labels_added(const char *obj, int index, void *ctx) {
688 : (void)index;
689 0 : struct history_ctx *hc = ctx;
690 :
691 : /* Gmail history item: {"message": {"id": "..."}, "labelIds": [...]} */
692 0 : char *id = json_get_nested_string(obj, "message", "id");
693 0 : if (!id) return;
694 :
695 0 : char **add_labels = NULL;
696 0 : int add_count = 0;
697 0 : json_get_string_array(obj, "labelIds", &add_labels, &add_count);
698 :
699 0 : for (int i = 0; i < add_count; i++) {
700 0 : if (gmail_sync_is_filtered_label(add_labels[i])) continue;
701 0 : const char *idx_name = add_labels[i];
702 0 : if (strcmp(add_labels[i], "SPAM") == 0) idx_name = "_spam";
703 0 : else if (strcmp(add_labels[i], "TRASH") == 0) idx_name = "_trash";
704 0 : label_idx_add(idx_name, id);
705 : /* Only remove from _nolabel when a real (non-CATEGORY_) label is added */
706 0 : if (!is_category_label(add_labels[i]))
707 0 : label_idx_remove("_nolabel", id);
708 : }
709 :
710 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate. */
711 0 : local_hdr_update_labels("", id,
712 : (const char **)add_labels, add_count, NULL, 0);
713 :
714 0 : for (int i = 0; i < add_count; i++) free(add_labels[i]);
715 0 : free(add_labels);
716 0 : free(id);
717 0 : hc->label_changes++;
718 : }
719 :
720 0 : static void process_labels_removed(const char *obj, int index, void *ctx) {
721 : (void)index;
722 0 : struct history_ctx *hc = ctx;
723 :
724 : /* Gmail history item: {"message": {"id": "..."}, "labelIds": [...]} */
725 0 : char *id = json_get_nested_string(obj, "message", "id");
726 0 : if (!id) return;
727 :
728 0 : char **rm_labels = NULL;
729 0 : int rm_count = 0;
730 0 : json_get_string_array(obj, "labelIds", &rm_labels, &rm_count);
731 :
732 0 : for (int i = 0; i < rm_count; i++) {
733 0 : if (gmail_sync_is_filtered_label(rm_labels[i])) continue;
734 0 : const char *idx_name = rm_labels[i];
735 0 : if (strcmp(rm_labels[i], "SPAM") == 0) idx_name = "_spam";
736 0 : else if (strcmp(rm_labels[i], "TRASH") == 0) idx_name = "_trash";
737 0 : label_idx_remove(idx_name, id);
738 : }
739 :
740 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate.
741 : * Must be done BEFORE freeing rm_labels (used-after-free guard). */
742 0 : local_hdr_update_labels("", id,
743 : NULL, 0, (const char **)rm_labels, rm_count);
744 :
745 0 : for (int i = 0; i < rm_count; i++) free(rm_labels[i]);
746 0 : free(rm_labels);
747 :
748 : /* Check if any labels remain; if none → add to _nolabel */
749 : /* Quick check: fetch message labels from server */
750 0 : char **cur_labels = NULL;
751 0 : int cur_count = 0;
752 0 : char *raw = gmail_fetch_message(hc->gc, id, &cur_labels, &cur_count);
753 0 : free(raw);
754 :
755 0 : int has_real_label = 0;
756 0 : for (int i = 0; i < cur_count; i++) {
757 0 : if (!gmail_sync_is_filtered_label(cur_labels[i]) &&
758 0 : !is_category_label(cur_labels[i]))
759 0 : has_real_label = 1;
760 0 : free(cur_labels[i]);
761 : }
762 0 : free(cur_labels);
763 :
764 0 : if (!has_real_label) {
765 0 : label_idx_add("_nolabel", id);
766 : /* Archived messages are always read: clear UNSEEN from .hdr flags */
767 0 : char *cur_hdr = local_hdr_load("", id);
768 0 : if (cur_hdr) {
769 0 : char *last_tab = strrchr(cur_hdr, '\t');
770 0 : if (last_tab) {
771 0 : int cur_flags = atoi(last_tab + 1);
772 0 : if (cur_flags & MSG_FLAG_UNSEEN)
773 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
774 : }
775 0 : free(cur_hdr);
776 : }
777 : }
778 :
779 0 : free(id);
780 0 : hc->label_changes++;
781 : }
782 :
783 : /* ── One-time repair: archived messages must not be unread ─────────── */
784 :
785 7 : void gmail_sync_repair_archive_flags(void) {
786 7 : char (*uids)[17] = NULL;
787 7 : int count = 0;
788 7 : if (label_idx_load("_nolabel", &uids, &count) != 0 || count == 0) {
789 7 : free(uids);
790 7 : return;
791 : }
792 0 : for (int i = 0; i < count; i++) {
793 0 : char *hdr = local_hdr_load("", uids[i]);
794 0 : if (!hdr) continue;
795 0 : char *last_tab = strrchr(hdr, '\t');
796 0 : if (last_tab) {
797 0 : int flags = atoi(last_tab + 1);
798 0 : if (flags & MSG_FLAG_UNSEEN)
799 0 : local_hdr_update_flags("", uids[i], flags & ~MSG_FLAG_UNSEEN);
800 : }
801 0 : free(hdr);
802 : }
803 0 : free(uids);
804 : }
805 :
806 : /* ── History record dispatcher ───────────────────────────────────── */
807 :
808 : /* Called once per entry in the "history" array.
809 : * Each record may contain messagesAdded/Deleted and labelsAdded/Removed. */
810 1 : static void process_history_record(const char *rec, int index, void *ctx) {
811 : (void)index;
812 1 : struct history_ctx *hc = ctx;
813 1 : json_foreach_object(rec, "messagesAdded", process_message_added, hc);
814 1 : json_foreach_object(rec, "messagesDeleted", process_message_deleted, hc);
815 1 : json_foreach_object(rec, "labelsAdded", process_labels_added, hc);
816 1 : json_foreach_object(rec, "labelsRemoved", process_labels_removed, hc);
817 1 : }
818 :
819 : /* ── Recover stale UNREAD labels from server ─────────────────────── */
820 :
821 : /* Query the server for currently unread messages and update any locally
822 : * cached .hdr files that are missing the UNREAD label. Called after
823 : * every sync to correct state that the (previously broken) incremental
824 : * sync may have failed to propagate. */
825 7 : static void recover_unread_labels(GmailClient *gc) {
826 7 : char (*server_uids)[17] = NULL;
827 7 : int server_count = 0;
828 7 : char *dummy_hid = NULL;
829 :
830 7 : if (gmail_list_messages(gc, "UNREAD", NULL,
831 : &server_uids, &server_count, &dummy_hid) != 0) {
832 0 : free(dummy_hid);
833 0 : return;
834 : }
835 7 : free(dummy_hid);
836 :
837 7 : if (server_count == 0) { free(server_uids); return; }
838 :
839 7 : logger_log(LOG_INFO, "gmail_sync: recover_unread: %d unread on server",
840 : server_count);
841 :
842 : /* Load current local UNREAD index to skip already-correct entries */
843 7 : char (*local_uids)[17] = NULL;
844 7 : int local_count = 0;
845 7 : label_idx_load("UNREAD", &local_uids, &local_count);
846 :
847 7 : MailRules *rules = mail_rules_load(local_store_account_name());
848 7 : int updated = 0, fetched = 0;
849 :
850 112 : for (int i = 0; i < server_count; i++) {
851 105 : const char *uid = server_uids[i];
852 :
853 : /* Binary search in sorted local UNREAD index */
854 105 : int found = 0;
855 105 : int lo = 0, hi = local_count - 1;
856 422 : while (lo <= hi) {
857 422 : int mid = lo + (hi - lo) / 2;
858 422 : int cmp = strcmp(local_uids[mid], uid);
859 422 : if (cmp == 0) { found = 1; break; }
860 317 : if (cmp < 0) lo = mid + 1; else hi = mid - 1;
861 : }
862 105 : if (found) continue; /* already in local UNREAD index */
863 :
864 0 : if (local_hdr_exists("", uid)) {
865 : /* Cached but missing UNREAD label — patch the .hdr */
866 0 : const char *add[] = {"UNREAD"};
867 0 : local_hdr_update_labels("", uid, add, 1, NULL, 0);
868 0 : label_idx_add("UNREAD", uid);
869 0 : updated++;
870 : } else {
871 : /* Not in local cache — download with current labels (includes UNREAD) */
872 0 : if (store_fetched_message(gc, uid, rules) == 0)
873 0 : fetched++;
874 : }
875 : }
876 :
877 7 : mail_rules_free(rules);
878 7 : free(server_uids);
879 7 : free(local_uids);
880 :
881 7 : if (updated > 0 || fetched > 0) {
882 0 : logger_log(LOG_INFO,
883 : "gmail_sync: recover_unread: patched=%d fetched=%d",
884 : updated, fetched);
885 0 : if (updated > 0)
886 0 : fprintf(stderr, " Recovered UNREAD flag for %d cached messages.\n",
887 : updated);
888 0 : if (fetched > 0)
889 0 : fprintf(stderr, " Downloaded %d new unread messages.\n", fetched);
890 : }
891 : }
892 :
893 : /* ── Incremental Sync ─────────────────────────────────────────────── */
894 :
895 9 : int gmail_sync_incremental(GmailClient *gc) {
896 9 : char *history_id = local_gmail_history_load();
897 9 : if (!history_id) {
898 0 : logger_log(LOG_INFO, "gmail_sync: no historyId, need full sync");
899 0 : return -2;
900 : }
901 :
902 9 : logger_log(LOG_INFO, "gmail_sync: incremental from historyId %s", history_id);
903 :
904 9 : char *resp = gmail_get_history(gc, history_id);
905 9 : free(history_id);
906 :
907 9 : if (!resp) {
908 2 : fprintf(stderr, " Incremental: History API returned error/404 (historyId expired or network issue).\n");
909 2 : logger_log(LOG_WARN, "gmail_sync: history expired or error");
910 2 : return -2; /* Signal: need full sync */
911 : }
912 :
913 7 : MailRules *inc_rules = mail_rules_load(local_store_account_name());
914 7 : struct history_ctx hc = { .gc = gc, .rules = inc_rules, .added = 0, .deleted = 0, .label_changes = 0 };
915 :
916 : /* Process each history record.
917 : * The response is {"history": [{...}, ...], "historyId": "..."}.
918 : * Each record may contain messagesAdded, messagesDeleted,
919 : * labelsAdded, labelsRemoved arrays.
920 : * process_history_record dispatches to the individual handlers. */
921 7 : json_foreach_object(resp, "history", process_history_record, &hc);
922 :
923 : /* Save updated historyId */
924 7 : RAII_STRING char *new_history_id = json_get_string(resp, "historyId");
925 7 : if (new_history_id)
926 7 : local_gmail_history_save(new_history_id);
927 :
928 7 : free(resp);
929 :
930 : /* Refresh label name mapping if any label events occurred */
931 7 : if (hc.label_changes > 0) {
932 0 : char **lbl_names = NULL, **lbl_ids = NULL;
933 0 : int lbl_count = 0;
934 0 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
935 0 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
936 0 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
937 0 : free(lbl_names);
938 0 : free(lbl_ids);
939 : }
940 : }
941 :
942 : /* Ensure no archived message is marked unread (repair existing data too) */
943 7 : gmail_sync_repair_archive_flags();
944 :
945 : /* Refresh UNREAD state from the server to correct any label changes that
946 : * were missed while the history-parsing bug was present. */
947 7 : recover_unread_labels(gc);
948 :
949 7 : mail_rules_free(inc_rules);
950 :
951 : /* Report what actually changed — "up to date" alone would hide the
952 : * messages this run downloaded. */
953 7 : if (hc.added || hc.deleted || hc.label_changes)
954 1 : fprintf(stderr, " Incremental sync: %d new, %d deleted, %d label change(s).\n",
955 : hc.added, hc.deleted, hc.label_changes);
956 : else
957 6 : fprintf(stderr, " Incremental sync: up to date.\n");
958 :
959 7 : logger_log(LOG_INFO, "gmail_sync: incremental done — added=%d deleted=%d labels=%d",
960 : hc.added, hc.deleted, hc.label_changes);
961 7 : return 0;
962 : }
963 :
964 : /* ── Auto Sync (public entry point) ───────────────────────────────── */
965 :
966 : /**
967 : * Smart sync flow:
968 : *
969 : * 1. If pending_fetch.tsv is non-empty, download those first (resuming an
970 : * interrupted previous sync or initial download).
971 : * 2. If the local store was already complete (no pending at start) AND a
972 : * valid historyId exists, use the fast incremental path (1–2 API calls).
973 : * 3. Otherwise, run reconcile (full UID listing) to discover any missing
974 : * messages, then download them.
975 : *
976 : * This ensures:
977 : * - First run or expired historyId: O(N) reconcile → O(missing) downloads.
978 : * - Subsequent runs on a mature store: O(1) incremental (no listing at all).
979 : * - Interrupted downloads: resume from pending_fetch.tsv without re-listing.
980 : */
981 68 : int gmail_sync(GmailClient *gc) {
982 : /* Step 1: check readiness before downloading anything */
983 68 : int had_pending = local_pending_fetch_count() > 0;
984 :
985 : /* Step 2: drain any queued downloads from a previous (possibly interrupted) sync */
986 68 : if (had_pending)
987 3 : gmail_sync_fetch_pending(gc);
988 :
989 : /* Step 3: try fast incremental path if we have a saved historyId.
990 : * We do this regardless of whether there were pending downloads —
991 : * draining pending_fetch.tsv already brought the local store up to the
992 : * reconcile snapshot; incremental then catches anything that arrived
993 : * on the server after that snapshot. */
994 68 : char *history_id = local_gmail_history_load();
995 68 : int have_history = (history_id != NULL);
996 68 : free(history_id);
997 :
998 68 : if (have_history) {
999 9 : fprintf(stderr, " Incremental sync (historyId present)...\n");
1000 9 : int rc = gmail_sync_incremental(gc);
1001 9 : if (rc == 0)
1002 7 : return 0; /* fast path — done; the callee reported the outcome */
1003 2 : if (rc != -2) return rc; /* unexpected error */
1004 2 : fprintf(stderr, " Incremental sync: historyId expired — falling back to full reconcile.\n");
1005 2 : logger_log(LOG_INFO, "gmail_sync: historyId expired, falling back to reconcile");
1006 : } else {
1007 59 : fprintf(stderr, " No saved historyId — full reconcile needed.\n");
1008 : }
1009 :
1010 : /* Step 4: reconcile (discover what is missing) */
1011 61 : int queued = gmail_sync_reconcile(gc);
1012 61 : if (queued < 0) return -1;
1013 :
1014 : /* Step 5: download what reconcile found */
1015 61 : if (queued > 0)
1016 52 : gmail_sync_fetch_pending(gc);
1017 :
1018 : /* Step 6: rebuild label indexes from .hdr files.
1019 : * Necessary when all messages were already cached (queued == 0) but
1020 : * label .idx files were deleted or are missing (e.g. manual deletion
1021 : * or upgrade from an older version). */
1022 : {
1023 61 : char (*all_uids)[17] = NULL;
1024 61 : int all_count = 0;
1025 61 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
1026 61 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
1027 61 : free(all_uids);
1028 : }
1029 :
1030 61 : return 0;
1031 : }
|
