Line data Source code
1 : #include "gmail_sync.h"
2 : #include "gmail_client.h"
3 : #include "local_store.h"
4 : #include "mail_rules.h"
5 : #include "mime_util.h"
6 : #include "json_util.h"
7 : #include "logger.h"
8 : #include "raii.h"
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <string.h>
12 : #include <stddef.h>
13 :
14 : /* ── Progress callbacks ───────────────────────────────────────────── */
15 :
16 : /* Called by gmail_list_messages after each page: cur = messages collected so far */
17 75 : static void list_progress_cb(size_t cur, size_t total, void *ctx) {
18 : (void)total; (void)ctx;
19 75 : fprintf(stderr, "\r\033[K Listing messages... %zu found", cur);
20 75 : fflush(stderr);
21 75 : }
22 :
23 : /* ── Gmail .hdr file format ───────────────────────────────────────── */
24 :
25 : /**
26 : * Build a .hdr string from raw message headers and label list.
27 : * Format: from\tsubject\tdate\tlabel1,label2,...\tflags\n
28 : *
29 : * Returns heap-allocated string. Caller must free().
30 : */
31 829 : char *gmail_sync_build_hdr(const char *raw_msg, char **labels, int label_count) {
32 1658 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
33 1658 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
34 1658 : RAII_STRING char *date_raw = mime_get_header(raw_msg, "Date");
35 :
36 1658 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
37 1658 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
38 1658 : RAII_STRING char *date_fmt = date_raw ? mime_format_date(date_raw) : NULL;
39 :
40 829 : const char *from = from_dec ? from_dec : "";
41 829 : const char *subj = subj_dec ? subj_dec : "";
42 829 : const char *date = date_fmt ? date_fmt : "";
43 :
44 : /* Build comma-separated label string */
45 829 : size_t lbl_len = 1;
46 2253 : for (int i = 0; i < label_count; i++)
47 1424 : lbl_len += strlen(labels[i]) + 1;
48 829 : char *lbl_str = calloc(lbl_len, 1);
49 829 : if (lbl_str) {
50 2253 : for (int i = 0; i < label_count; i++) {
51 1424 : if (i > 0) strcat(lbl_str, ",");
52 1424 : strcat(lbl_str, labels[i]);
53 : }
54 : }
55 :
56 : /* Compute flags bitmask from labels */
57 829 : int flags = 0;
58 2253 : for (int i = 0; i < label_count; i++) {
59 1424 : if (strcmp(labels[i], "UNREAD") == 0) flags |= MSG_FLAG_UNSEEN;
60 1424 : if (strcmp(labels[i], "STARRED") == 0) flags |= MSG_FLAG_FLAGGED;
61 1424 : if (strcmp(labels[i], "SPAM") == 0) flags |= MSG_FLAG_JUNK;
62 : }
63 :
64 : /* Compute attachment and DMARC flags from the raw message so the TUI
65 : * never needs to download anything to display these indicators. */
66 1658 : RAII_STRING char *ct_raw = mime_get_header(raw_msg, "Content-Type");
67 829 : if (ct_raw && strcasestr(ct_raw, "multipart/mixed"))
68 58 : flags |= MSG_FLAG_ATTACH;
69 829 : flags |= MSG_FLAG_ATTACH_CHECKED;
70 :
71 829 : RAII_STRING char *ar_raw = mime_get_header(raw_msg, "Authentication-Results");
72 829 : int dmarc_st = mime_get_dmarc_status(ar_raw);
73 829 : if (dmarc_st == 1) flags |= MSG_FLAG_DMARC_PASS;
74 829 : else if (dmarc_st == -1) flags |= MSG_FLAG_DMARC_FAIL;
75 829 : if (dmarc_st != -2) flags |= MSG_FLAG_DMARC_CHECKED;
76 :
77 : /* Replace tabs in fields with spaces */
78 829 : char *hdr = NULL;
79 829 : if (asprintf(&hdr, "%s\t%s\t%s\t%s\t%d",
80 : from, subj, date, lbl_str ? lbl_str : "", flags) == -1)
81 0 : hdr = NULL;
82 829 : free(lbl_str);
83 :
84 : /* Sanitise: replace any tabs within field values */
85 829 : if (hdr) {
86 : /* The first 4 tabs are field separators; tabs within values got
87 : * inserted by asprintf if field values contained tabs. Since we
88 : * used tab as separator this is inherently safe (mime_decode_words
89 : * doesn't produce tabs), but defend anyway. */
90 : }
91 :
92 829 : return hdr;
93 : }
94 :
95 : /* ── Filtered labels (metadata-only, excluded from indexing) ──────── */
96 :
97 7814 : int gmail_sync_is_filtered_label(const char *label_id) {
98 7814 : if (!label_id) return 1;
99 7813 : if (strcmp(label_id, "IMPORTANT") == 0) return 1;
100 7811 : if (strcmp(label_id, "CHAT") == 0) return 1;
101 7811 : return 0;
102 : }
103 :
104 : /* Returns 1 if label_id is a Gmail automatic inbox category (CATEGORY_*).
105 : * Category labels are indexed like user labels, but a message whose ONLY
106 : * non-filtered labels are CATEGORY_* is also added to _nolabel (Archive). */
107 7786 : static int is_category_label(const char *label_id) {
108 7786 : return label_id && strncmp(label_id, "CATEGORY_", 9) == 0;
109 : }
110 :
111 : /* ── Mail rules helper ───────────────────────────────────────────── */
112 :
113 : /* Apply mail rules to a newly stored message.
114 : * Builds labels_csv from Gmail label IDs (resolving user labels to names),
115 : * calls mail_rules_apply(), then updates the local .hdr and label indexes. */
116 822 : static void apply_rules_to_new_message(const MailRules *rules, const char *uid,
117 : const char *raw_msg,
118 : char **labels, int label_count)
119 : {
120 822 : if (!rules || rules->count == 0) return;
121 :
122 2 : RAII_STRING char *from_raw = mime_get_header(raw_msg, "From");
123 2 : RAII_STRING char *subj_raw = mime_get_header(raw_msg, "Subject");
124 2 : RAII_STRING char *to_raw = mime_get_header(raw_msg, "To");
125 2 : RAII_STRING char *from_dec = from_raw ? mime_decode_words(from_raw) : NULL;
126 2 : RAII_STRING char *subj_dec = subj_raw ? mime_decode_words(subj_raw) : NULL;
127 2 : RAII_STRING char *to_dec = to_raw ? mime_decode_words(to_raw) : NULL;
128 :
129 : /* Build labels_csv using friendly names where available */
130 1 : size_t lcsz = 1;
131 3 : for (int i = 0; i < label_count; i++) {
132 2 : char *name = local_gmail_label_name_lookup(labels[i]);
133 2 : lcsz += strlen(name ? name : labels[i]) + 2;
134 2 : free(name);
135 : }
136 1 : char *lcsv = calloc(lcsz, 1);
137 1 : if (!lcsv) return;
138 3 : for (int i = 0; i < label_count; i++) {
139 2 : char *name = local_gmail_label_name_lookup(labels[i]);
140 2 : const char *display = name ? name : labels[i];
141 2 : if (lcsv[0]) strcat(lcsv, ",");
142 2 : strcat(lcsv, display);
143 2 : free(name);
144 : }
145 :
146 1 : char **add_out = NULL; int add_count = 0;
147 1 : char **rm_out = NULL; int rm_count = 0;
148 1 : int fired = mail_rules_apply(rules,
149 : from_dec, subj_dec, to_dec, lcsv,
150 : NULL, (time_t)0, /* body/date unavailable during Gmail sync */
151 : &add_out, &add_count,
152 : &rm_out, &rm_count);
153 1 : free(lcsv);
154 1 : if (fired <= 0) return;
155 :
156 1 : logger_log(LOG_INFO, "gmail_sync: rules fired=%d for %s (add=%d rm=%d)",
157 : fired, uid, add_count, rm_count);
158 :
159 : /* Update local .hdr and label indexes */
160 1 : local_hdr_update_labels("", uid,
161 : (const char **)add_out, add_count,
162 : (const char **)rm_out, rm_count);
163 2 : for (int i = 0; i < add_count; i++) {
164 1 : label_idx_add(add_out[i], uid);
165 1 : free(add_out[i]);
166 : }
167 1 : for (int i = 0; i < rm_count; i++) {
168 0 : label_idx_remove(rm_out[i], uid);
169 0 : free(rm_out[i]);
170 : }
171 1 : free(add_out);
172 1 : free(rm_out);
173 :
174 : /* Update contact suggestion cache */
175 : {
176 1 : char *from_h = mime_get_header(raw_msg, "From");
177 1 : char *to_h = mime_get_header(raw_msg, "To");
178 1 : char *cc_h = mime_get_header(raw_msg, "Cc");
179 1 : local_contacts_update(from_h, to_h, cc_h);
180 1 : free(from_h); free(to_h); free(cc_h);
181 : }
182 : }
183 :
184 : /* ── Label index rebuild ──────────────────────────────────────────── */
185 :
186 : typedef struct { char label[64]; char uid[17]; } LabelUidPair;
187 :
188 40796 : static int cmp_lbl_uid_pair(const void *a, const void *b) {
189 40796 : const LabelUidPair *pa = a, *pb = b;
190 40796 : int c = strcmp(pa->label, pb->label);
191 40796 : return c ? c : strcmp(pa->uid, pb->uid);
192 : }
193 :
194 : /**
195 : * Rebuild ALL label .idx files from the .hdr files for the given UIDs.
196 : *
197 : * Efficient O(N log N) approach:
198 : * 1. Read each .hdr and collect (label, uid) pairs in memory.
199 : * 2. Sort the flat pair array.
200 : * 3. Write each label's .idx file in one grouped pass.
201 : *
202 : * This is called at the end of every full sync so that cached messages
203 : * (whose .idx entries were never written) are correctly indexed.
204 : */
205 79 : static void rebuild_label_indexes(const char (*uids)[17], int uid_count) {
206 79 : if (uid_count <= 0) return;
207 :
208 79 : fprintf(stderr, " Rebuilding label indexes...");
209 79 : fflush(stderr);
210 :
211 : /* Phase 1: collect (label, uid) pairs from all .hdr files */
212 79 : size_t cap = (size_t)uid_count * 5; /* ~5 labels per message */
213 79 : LabelUidPair *pairs = malloc(cap * sizeof(LabelUidPair));
214 79 : if (!pairs) {
215 0 : fprintf(stderr, " [out of memory]\n");
216 0 : return;
217 : }
218 79 : int npairs = 0;
219 :
220 3967 : for (int i = 0; i < uid_count; i++) {
221 : /* Load full .hdr so we can both collect label pairs and sync the
222 : * flags integer in one read (avoid a separate local_hdr_get_labels
223 : * call followed by local_hdr_update_labels). */
224 3888 : char *hdr = local_hdr_load("", uids[i]);
225 3888 : if (!hdr) continue;
226 :
227 : /* Locate labels field (4th tab-separated token).
228 : * Track the tab pointer so we can NUL-terminate the prefix later. */
229 3888 : char *t3_tab = hdr;
230 15549 : for (int f = 0; f < 3; f++) {
231 11662 : t3_tab = strchr(t3_tab, '\t');
232 11662 : if (!t3_tab) break;
233 11661 : if (f < 2) t3_tab++;
234 : }
235 3888 : if (!t3_tab || t3_tab == hdr) { free(hdr); continue; }
236 3887 : char *lbl_start = t3_tab + 1; /* start of labels CSV field */
237 :
238 : /* Locate optional flags field (5th token) and read old value */
239 3887 : char *t4 = strchr(lbl_start, '\t');
240 3887 : int old_flags = 0;
241 3887 : if (t4) {
242 3887 : old_flags = atoi(t4 + 1);
243 3887 : *t4 = '\0'; /* NUL-terminate labels field in-place */
244 : } else {
245 0 : char *nl = strchr(lbl_start, '\n');
246 0 : if (nl) *nl = '\0';
247 : }
248 :
249 : /* Derive new flags from labels (preserving non-label bits) */
250 3887 : int new_flags = old_flags & ~(MSG_FLAG_UNSEEN | MSG_FLAG_FLAGGED);
251 3887 : int has_real = 0;
252 :
253 : /* Iterate labels via a copy (tokenising modifies the string) */
254 3887 : char *lbl_copy = strdup(lbl_start);
255 3887 : if (!lbl_copy) { free(hdr); continue; }
256 :
257 3887 : char *tok = lbl_copy;
258 10273 : while (tok) {
259 6386 : char *comma = strchr(tok, ',');
260 6386 : if (comma) *comma = '\0';
261 :
262 6386 : if (tok[0] && !gmail_sync_is_filtered_label(tok)) {
263 6376 : const char *idx_name = tok;
264 6376 : if (strcmp(tok, "SPAM") == 0) idx_name = "_spam";
265 6370 : else if (strcmp(tok, "TRASH") == 0) idx_name = "_trash";
266 :
267 6376 : if (npairs >= (int)cap) {
268 0 : cap = cap * 2 + 1;
269 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
270 0 : if (!tmp) { free(lbl_copy); free(hdr); free(pairs); return; }
271 0 : pairs = tmp;
272 : }
273 6376 : strncpy(pairs[npairs].label, idx_name, 63);
274 6376 : pairs[npairs].label[63] = '\0';
275 6376 : strncpy(pairs[npairs].uid, uids[i], 16);
276 6376 : pairs[npairs].uid[16] = '\0';
277 6376 : npairs++;
278 :
279 6376 : if (!is_category_label(tok)) has_real = 1;
280 6376 : if (strcmp(tok, "UNREAD") == 0) new_flags |= MSG_FLAG_UNSEEN;
281 6376 : if (strcmp(tok, "STARRED") == 0) new_flags |= MSG_FLAG_FLAGGED;
282 : }
283 6386 : tok = comma ? comma + 1 : NULL;
284 : }
285 3887 : free(lbl_copy);
286 :
287 : /* Messages with no real (non-CATEGORY_) label → Archive.
288 : * Archived messages are always considered read. */
289 3887 : if (!has_real) {
290 15 : if (npairs >= (int)cap) {
291 0 : cap = cap * 2 + 1;
292 0 : LabelUidPair *tmp = realloc(pairs, cap * sizeof(LabelUidPair));
293 0 : if (!tmp) { free(hdr); free(pairs); return; }
294 0 : pairs = tmp;
295 : }
296 15 : strncpy(pairs[npairs].label, "_nolabel", 63);
297 15 : strncpy(pairs[npairs].uid, uids[i], 16);
298 15 : pairs[npairs].uid[16] = '\0';
299 15 : npairs++;
300 15 : new_flags &= ~MSG_FLAG_UNSEEN;
301 : }
302 :
303 : /* Sync flags integer if it disagrees with the labels CSV.
304 : * lbl_start still points into hdr at the labels field.
305 : * NUL-terminate the prefix at t3_tab then reassemble. */
306 3887 : if (new_flags != old_flags) {
307 3 : *t3_tab = '\0';
308 3 : char *updated = NULL;
309 3 : if (asprintf(&updated, "%s\t%s\t%d", hdr, lbl_start, new_flags) != -1) {
310 3 : local_hdr_save("", uids[i], updated, strlen(updated));
311 3 : free(updated);
312 : }
313 : }
314 :
315 3887 : free(hdr);
316 : }
317 :
318 : /* Phase 2: sort by (label, uid) */
319 79 : qsort(pairs, (size_t)npairs, sizeof(LabelUidPair), cmp_lbl_uid_pair);
320 :
321 : /* Phase 3: group by label and write each .idx file */
322 79 : int labels_written = 0;
323 79 : int i = 0;
324 331 : while (i < npairs) {
325 252 : const char *cur_label = pairs[i].label;
326 252 : int j = i;
327 6643 : while (j < npairs && strcmp(pairs[j].label, cur_label) == 0) j++;
328 252 : int run = j - i;
329 :
330 252 : char (*uid_arr)[17] = malloc((size_t)run * sizeof(char[17]));
331 252 : if (uid_arr) {
332 252 : int unique = 0;
333 6643 : for (int k = i; k < j; k++) {
334 6391 : if (unique == 0 ||
335 6139 : strcmp(uid_arr[unique - 1], pairs[k].uid) != 0) {
336 6391 : memcpy(uid_arr[unique++], pairs[k].uid, 17);
337 : }
338 : }
339 252 : label_idx_write(cur_label, (const char (*)[17])uid_arr, unique);
340 252 : free(uid_arr);
341 252 : labels_written++;
342 : }
343 252 : i = j;
344 : }
345 79 : free(pairs);
346 :
347 79 : fprintf(stderr, "\r\033[K Label indexes rebuilt (%d labels)\n",
348 : labels_written);
349 79 : logger_log(LOG_INFO,
350 : "gmail_sync: rebuilt %d label indexes from %d messages",
351 : labels_written, uid_count);
352 : }
353 :
354 : /**
355 : * Rebuild all label .idx files from locally cached .hdr files.
356 : * Does NOT contact the Gmail API.
357 : * Use this to repair missing or incomplete indexes without re-downloading.
358 : */
359 16 : int gmail_sync_rebuild_indexes(void) {
360 16 : char (*uids)[17] = NULL;
361 16 : int count = 0;
362 16 : if (local_hdr_list_all_uids("", &uids, &count) != 0) {
363 0 : fprintf(stderr, "Error: could not scan local message store.\n");
364 0 : return -1;
365 : }
366 16 : fprintf(stderr, " Found %d cached messages.\n", count);
367 16 : rebuild_label_indexes((const char (*)[17])uids, count);
368 16 : free(uids);
369 16 : return 0;
370 : }
371 :
372 : /* ── Single message fetch+store helper ───────────────────────────────── */
373 :
374 : /**
375 : * Fetch one message from the Gmail API, save .eml + .hdr, apply rules,
376 : * update label indexes.
377 : * Returns 0 on success, -1 on fetch error (transient; caller should retry).
378 : */
379 820 : static int store_fetched_message(GmailClient *gc, const char *uid,
380 : const MailRules *rules)
381 : {
382 820 : char **labels = NULL;
383 820 : int label_count = 0;
384 820 : char *raw = gmail_fetch_message(gc, uid, &labels, &label_count);
385 820 : if (!raw) {
386 1 : logger_log(LOG_WARN, "gmail_sync: failed to fetch %s", uid);
387 1 : for (int j = 0; j < label_count; j++) free(labels[j]);
388 1 : free(labels);
389 1 : return -1;
390 : }
391 :
392 819 : local_msg_save("", uid, raw, strlen(raw));
393 :
394 819 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
395 819 : if (hdr) { local_hdr_save("", uid, hdr, strlen(hdr)); free(hdr); }
396 :
397 819 : apply_rules_to_new_message(rules, uid, raw, labels, label_count);
398 819 : free(raw);
399 :
400 819 : int has_real_label = 0;
401 2227 : for (int j = 0; j < label_count; j++) {
402 1408 : if (gmail_sync_is_filtered_label(labels[j])) continue;
403 1407 : const char *idx_name = labels[j];
404 1407 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
405 1407 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
406 1407 : label_idx_add(idx_name, uid);
407 1407 : if (!is_category_label(labels[j])) has_real_label = 1;
408 : }
409 819 : if (!has_real_label) {
410 1 : label_idx_add("_nolabel", uid);
411 1 : int cur_flags = 0;
412 3 : for (int j = 0; j < label_count; j++) {
413 2 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
414 2 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
415 : }
416 1 : if (cur_flags & MSG_FLAG_UNSEEN)
417 0 : local_hdr_update_flags("", uid, cur_flags & ~MSG_FLAG_UNSEEN);
418 : }
419 :
420 2227 : for (int j = 0; j < label_count; j++) free(labels[j]);
421 819 : free(labels);
422 819 : return 0;
423 : }
424 :
425 : /* ── Reconcile: discover missing UIDs and queue them ─────────────────── */
426 :
427 : /**
428 : * List all server-side message IDs, compare with the local store, and
429 : * add any missing UIDs to pending_fetch.tsv. Does NOT download messages.
430 : *
431 : * Also updates the historyId and label name mapping so subsequent
432 : * incremental syncs know where to resume from.
433 : *
434 : * Returns the number of UIDs added to the pending-fetch queue, or -1 on
435 : * a fatal error (e.g. the server cannot be reached).
436 : */
437 68 : int gmail_sync_reconcile(GmailClient *gc) {
438 68 : logger_log(LOG_INFO, "gmail_sync: reconcile — listing server messages");
439 :
440 68 : fprintf(stderr, " Listing messages...");
441 68 : fflush(stderr);
442 68 : gmail_set_progress(gc, list_progress_cb, NULL);
443 :
444 68 : char (*all_uids)[17] = NULL;
445 68 : int uid_count = 0;
446 68 : char *list_history_id = NULL;
447 68 : if (gmail_list_messages(gc, NULL, NULL, &all_uids, &uid_count, &list_history_id) != 0) {
448 0 : gmail_set_progress(gc, NULL, NULL);
449 0 : free(list_history_id);
450 0 : logger_log(LOG_ERROR, "gmail_sync: reconcile failed to list messages");
451 0 : return -1;
452 : }
453 68 : gmail_set_progress(gc, NULL, NULL);
454 68 : fprintf(stderr, "\r\033[K %d messages on server\n", uid_count);
455 :
456 : /* Clear any stale pending_fetch entries before repopulating */
457 68 : local_pending_fetch_clear();
458 :
459 68 : int queued = 0, cached = 0;
460 2112 : for (int i = 0; i < uid_count; i++) {
461 2044 : const char *uid = all_uids[i];
462 2044 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
463 1225 : cached++;
464 1225 : if (i % 500 == 0 || i == uid_count - 1) {
465 21 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
466 : i + 1, uid_count);
467 21 : fflush(stderr);
468 : }
469 1225 : continue;
470 : }
471 819 : local_pending_fetch_add(uid);
472 819 : queued++;
473 819 : if ((cached + queued) % 500 == 0 || i == uid_count - 1) {
474 55 : fprintf(stderr, "\r\033[K Scanning local store: %d/%d",
475 : i + 1, uid_count);
476 55 : fflush(stderr);
477 : }
478 : }
479 68 : if (uid_count > 0)
480 65 : fprintf(stderr, "\r\033[K %d cached, %d queued for download\n",
481 : cached, queued);
482 :
483 : /* Save historyId so next run can use incremental sync.
484 : * Prefer the historyId from the messages.list response (always fresh);
485 : * fall back to the /profile endpoint only if that field was absent. */
486 68 : if (list_history_id) {
487 65 : fprintf(stderr, " historyId from list response: %s\n", list_history_id);
488 65 : local_gmail_history_save(list_history_id);
489 65 : free(list_history_id);
490 65 : list_history_id = NULL;
491 : } else {
492 6 : RAII_STRING char *hid = gmail_get_history_id(gc);
493 3 : if (hid)
494 1 : local_gmail_history_save(hid);
495 : else
496 2 : logger_log(LOG_WARN, "gmail_sync: reconcile: could not retrieve historyId");
497 : }
498 :
499 : /* Save label ID→name mapping */
500 : {
501 68 : char **lbl_names = NULL, **lbl_ids = NULL;
502 68 : int lbl_count = 0;
503 68 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
504 66 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
505 631 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
506 66 : free(lbl_names); free(lbl_ids);
507 : }
508 : }
509 :
510 68 : free(all_uids);
511 68 : logger_log(LOG_INFO, "gmail_sync: reconcile done — %d cached, %d queued",
512 : cached, queued);
513 68 : return queued;
514 : }
515 :
516 : /* ── Fetch pending: download queued messages ─────────────────────────── */
517 :
518 : /**
519 : * Download all message UIDs listed in pending_fetch.tsv.
520 : * Removes each entry from the queue on successful download.
521 : * Leaves failures in the queue for retry on the next sync.
522 : *
523 : * Returns number of messages successfully downloaded.
524 : */
525 62 : int gmail_sync_fetch_pending(GmailClient *gc) {
526 62 : int count = 0;
527 62 : char (*uids)[17] = local_pending_fetch_load(&count);
528 62 : if (!uids || count == 0) {
529 1 : free(uids);
530 1 : return 0;
531 : }
532 :
533 61 : logger_log(LOG_INFO, "gmail_sync: fetch_pending — %d messages to download", count);
534 61 : fprintf(stderr, " Downloading %d message(s)...\n", count);
535 :
536 61 : MailRules *rules = mail_rules_load(local_store_account_name());
537 61 : int fetched = 0;
538 : #define PROGRESS_STEP 50
539 885 : for (int i = 0; i < count; i++) {
540 824 : const char *uid = uids[i];
541 :
542 824 : if (local_msg_exists("", uid) && local_hdr_exists("", uid)) {
543 : /* Already present — clean up stale pending entry */
544 4 : local_pending_fetch_remove(uid);
545 4 : continue;
546 : }
547 :
548 820 : if (store_fetched_message(gc, uid, rules) == 0) {
549 819 : local_pending_fetch_remove(uid);
550 819 : fetched++;
551 : }
552 : /* On failure: leave in queue for retry */
553 :
554 820 : if (i % PROGRESS_STEP == 0 || i == count - 1) {
555 118 : fprintf(stderr, "\r\033[K [%d/%d] downloaded", fetched, count);
556 118 : fflush(stderr);
557 : }
558 : }
559 61 : if (count > 0)
560 61 : fprintf(stderr, "\r\033[K %d of %d downloaded\n", fetched, count);
561 :
562 61 : mail_rules_free(rules);
563 61 : free(uids);
564 61 : logger_log(LOG_INFO, "gmail_sync: fetch_pending done — %d/%d downloaded",
565 : fetched, count);
566 61 : return fetched;
567 : }
568 :
569 : /* ── Full Sync ────────────────────────────────────────────────────── */
570 :
571 2 : int gmail_sync_full(GmailClient *gc) {
572 2 : logger_log(LOG_INFO, "gmail_sync: starting full sync");
573 :
574 2 : int queued = gmail_sync_reconcile(gc);
575 2 : if (queued < 0) return -1;
576 :
577 2 : if (queued > 0)
578 1 : gmail_sync_fetch_pending(gc);
579 :
580 : /* Rebuild label indexes from .hdr files so that even when all messages
581 : * were already cached (queued == 0) the indexes are consistent. */
582 : {
583 2 : char (*all_uids)[17] = NULL;
584 2 : int all_count = 0;
585 2 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
586 2 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
587 2 : free(all_uids);
588 : }
589 :
590 2 : return 0;
591 : }
592 :
593 : /* ── History delta processing ─────────────────────────────────────── */
594 :
595 : struct history_ctx {
596 : GmailClient *gc;
597 : MailRules *rules;
598 : int added;
599 : int deleted;
600 : int label_changes;
601 : };
602 :
603 3 : static void process_message_added(const char *obj, int index, void *ctx) {
604 : (void)index;
605 3 : struct history_ctx *hc = ctx;
606 :
607 : /* Gmail history item: {"message": {"id": "...", "labelIds": [...]}} */
608 3 : char *id = json_get_nested_string(obj, "message", "id");
609 3 : if (!id) return;
610 :
611 : /* Fetch and store the new message */
612 3 : char **labels = NULL;
613 3 : int label_count = 0;
614 3 : char *raw = gmail_fetch_message(hc->gc, id, &labels, &label_count);
615 3 : if (raw) {
616 3 : local_msg_save("", id, raw, strlen(raw));
617 :
618 3 : char *hdr = gmail_sync_build_hdr(raw, labels, label_count);
619 3 : if (hdr) {
620 3 : local_hdr_save("", id, hdr, strlen(hdr));
621 3 : free(hdr);
622 : }
623 :
624 3 : apply_rules_to_new_message(hc->rules, id, raw, labels, label_count);
625 3 : free(raw);
626 :
627 3 : int has_label = 0;
628 9 : for (int j = 0; j < label_count; j++) {
629 6 : if (gmail_sync_is_filtered_label(labels[j])) continue;
630 6 : const char *idx_name = labels[j];
631 6 : if (strcmp(labels[j], "SPAM") == 0) idx_name = "_spam";
632 6 : else if (strcmp(labels[j], "TRASH") == 0) idx_name = "_trash";
633 6 : label_idx_add(idx_name, id);
634 6 : has_label = 1;
635 : }
636 3 : if (!has_label) {
637 0 : label_idx_add("_nolabel", id);
638 : /* Archived messages are always read */
639 0 : int cur_flags = 0;
640 0 : for (int j = 0; j < label_count; j++) {
641 0 : if (strcmp(labels[j], "UNREAD") == 0) cur_flags |= MSG_FLAG_UNSEEN;
642 0 : if (strcmp(labels[j], "STARRED") == 0) cur_flags |= MSG_FLAG_FLAGGED;
643 : }
644 0 : if (cur_flags & MSG_FLAG_UNSEEN)
645 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
646 : }
647 :
648 3 : hc->added++;
649 : }
650 :
651 9 : for (int j = 0; j < label_count; j++) free(labels[j]);
652 3 : free(labels);
653 3 : free(id);
654 : }
655 :
656 1 : static void process_message_deleted(const char *obj, int index, void *ctx) {
657 : (void)index;
658 1 : struct history_ctx *hc = ctx;
659 :
660 : /* Gmail history item: {"message": {"id": "..."}} */
661 1 : char *id = json_get_nested_string(obj, "message", "id");
662 1 : if (!id) return;
663 :
664 1 : local_msg_delete("", id);
665 :
666 : /* Remove from all known label indexes — brute force scan */
667 : /* In practice this is rare; deleted messages are few */
668 1 : char **names = NULL, **ids = NULL;
669 1 : int count = 0;
670 1 : if (gmail_list_labels(hc->gc, &names, &ids, &count) == 0) {
671 4 : for (int i = 0; i < count; i++) {
672 3 : label_idx_remove(ids[i], id);
673 3 : free(names[i]);
674 3 : free(ids[i]);
675 : }
676 1 : free(names);
677 1 : free(ids);
678 : }
679 1 : label_idx_remove("_nolabel", id);
680 1 : label_idx_remove("_spam", id);
681 1 : label_idx_remove("_trash", id);
682 :
683 1 : hc->deleted++;
684 1 : free(id);
685 : }
686 :
687 1 : static void process_labels_added(const char *obj, int index, void *ctx) {
688 : (void)index;
689 1 : struct history_ctx *hc = ctx;
690 :
691 : /* Gmail history item: {"message": {"id": "..."}, "labelIds": [...]} */
692 1 : char *id = json_get_nested_string(obj, "message", "id");
693 1 : if (!id) return;
694 :
695 1 : char **add_labels = NULL;
696 1 : int add_count = 0;
697 1 : json_get_string_array(obj, "labelIds", &add_labels, &add_count);
698 :
699 2 : for (int i = 0; i < add_count; i++) {
700 1 : if (gmail_sync_is_filtered_label(add_labels[i])) continue;
701 1 : const char *idx_name = add_labels[i];
702 1 : if (strcmp(add_labels[i], "SPAM") == 0) idx_name = "_spam";
703 1 : else if (strcmp(add_labels[i], "TRASH") == 0) idx_name = "_trash";
704 1 : label_idx_add(idx_name, id);
705 : /* Only remove from _nolabel when a real (non-CATEGORY_) label is added */
706 1 : if (!is_category_label(add_labels[i]))
707 1 : label_idx_remove("_nolabel", id);
708 : }
709 :
710 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate. */
711 1 : local_hdr_update_labels("", id,
712 : (const char **)add_labels, add_count, NULL, 0);
713 :
714 2 : for (int i = 0; i < add_count; i++) free(add_labels[i]);
715 1 : free(add_labels);
716 1 : free(id);
717 1 : hc->label_changes++;
718 : }
719 :
720 1 : static void process_labels_removed(const char *obj, int index, void *ctx) {
721 : (void)index;
722 1 : struct history_ctx *hc = ctx;
723 :
724 : /* Gmail history item: {"message": {"id": "..."}, "labelIds": [...]} */
725 1 : char *id = json_get_nested_string(obj, "message", "id");
726 1 : if (!id) return;
727 :
728 1 : char **rm_labels = NULL;
729 1 : int rm_count = 0;
730 1 : json_get_string_array(obj, "labelIds", &rm_labels, &rm_count);
731 :
732 2 : for (int i = 0; i < rm_count; i++) {
733 1 : if (gmail_sync_is_filtered_label(rm_labels[i])) continue;
734 1 : const char *idx_name = rm_labels[i];
735 1 : if (strcmp(rm_labels[i], "SPAM") == 0) idx_name = "_spam";
736 1 : else if (strcmp(rm_labels[i], "TRASH") == 0) idx_name = "_trash";
737 1 : label_idx_remove(idx_name, id);
738 : }
739 :
740 : /* Keep .hdr labels field in sync so rebuild_label_indexes stays accurate.
741 : * Must be done BEFORE freeing rm_labels (used-after-free guard). */
742 1 : local_hdr_update_labels("", id,
743 : NULL, 0, (const char **)rm_labels, rm_count);
744 :
745 2 : for (int i = 0; i < rm_count; i++) free(rm_labels[i]);
746 1 : free(rm_labels);
747 :
748 : /* Check if any labels remain; if none → add to _nolabel */
749 : /* Quick check: fetch message labels from server */
750 1 : char **cur_labels = NULL;
751 1 : int cur_count = 0;
752 1 : char *raw = gmail_fetch_message(hc->gc, id, &cur_labels, &cur_count);
753 1 : free(raw);
754 :
755 1 : int has_real_label = 0;
756 3 : for (int i = 0; i < cur_count; i++) {
757 4 : if (!gmail_sync_is_filtered_label(cur_labels[i]) &&
758 2 : !is_category_label(cur_labels[i]))
759 2 : has_real_label = 1;
760 2 : free(cur_labels[i]);
761 : }
762 1 : free(cur_labels);
763 :
764 1 : if (!has_real_label) {
765 0 : label_idx_add("_nolabel", id);
766 : /* Archived messages are always read: clear UNSEEN from .hdr flags */
767 0 : char *cur_hdr = local_hdr_load("", id);
768 0 : if (cur_hdr) {
769 0 : char *last_tab = strrchr(cur_hdr, '\t');
770 0 : if (last_tab) {
771 0 : int cur_flags = atoi(last_tab + 1);
772 0 : if (cur_flags & MSG_FLAG_UNSEEN)
773 0 : local_hdr_update_flags("", id, cur_flags & ~MSG_FLAG_UNSEEN);
774 : }
775 0 : free(cur_hdr);
776 : }
777 : }
778 :
779 1 : free(id);
780 1 : hc->label_changes++;
781 : }
782 :
783 : /* ── One-time repair: archived messages must not be unread ─────────── */
784 :
785 11 : void gmail_sync_repair_archive_flags(void) {
786 11 : char (*uids)[17] = NULL;
787 11 : int count = 0;
788 11 : if (label_idx_load("_nolabel", &uids, &count) != 0 || count == 0) {
789 8 : free(uids);
790 8 : return;
791 : }
792 9 : for (int i = 0; i < count; i++) {
793 6 : char *hdr = local_hdr_load("", uids[i]);
794 6 : if (!hdr) continue;
795 6 : char *last_tab = strrchr(hdr, '\t');
796 6 : if (last_tab) {
797 6 : int flags = atoi(last_tab + 1);
798 6 : if (flags & MSG_FLAG_UNSEEN)
799 2 : local_hdr_update_flags("", uids[i], flags & ~MSG_FLAG_UNSEEN);
800 : }
801 6 : free(hdr);
802 : }
803 3 : free(uids);
804 : }
805 :
806 : /* ── History record dispatcher ───────────────────────────────────── */
807 :
808 : /* Called once per entry in the "history" array.
809 : * Each record may contain messagesAdded/Deleted and labelsAdded/Removed. */
810 2 : static void process_history_record(const char *rec, int index, void *ctx) {
811 : (void)index;
812 2 : struct history_ctx *hc = ctx;
813 2 : json_foreach_object(rec, "messagesAdded", process_message_added, hc);
814 2 : json_foreach_object(rec, "messagesDeleted", process_message_deleted, hc);
815 2 : json_foreach_object(rec, "labelsAdded", process_labels_added, hc);
816 2 : json_foreach_object(rec, "labelsRemoved", process_labels_removed, hc);
817 2 : }
818 :
819 : /* ── Recover stale UNREAD labels from server ─────────────────────── */
820 :
821 : /* Query the server for currently unread messages and update any locally
822 : * cached .hdr files that are missing the UNREAD label. Called after
823 : * every sync to correct state that the (previously broken) incremental
824 : * sync may have failed to propagate. */
825 8 : static void recover_unread_labels(GmailClient *gc) {
826 8 : char (*server_uids)[17] = NULL;
827 8 : int server_count = 0;
828 8 : char *dummy_hid = NULL;
829 :
830 8 : if (gmail_list_messages(gc, "UNREAD", NULL,
831 : &server_uids, &server_count, &dummy_hid) != 0) {
832 0 : free(dummy_hid);
833 1 : return;
834 : }
835 8 : free(dummy_hid);
836 :
837 8 : if (server_count == 0) { free(server_uids); return; }
838 :
839 7 : logger_log(LOG_INFO, "gmail_sync: recover_unread: %d unread on server",
840 : server_count);
841 :
842 : /* Load current local UNREAD index to skip already-correct entries */
843 7 : char (*local_uids)[17] = NULL;
844 7 : int local_count = 0;
845 7 : label_idx_load("UNREAD", &local_uids, &local_count);
846 :
847 7 : MailRules *rules = mail_rules_load(local_store_account_name());
848 7 : int updated = 0, fetched = 0;
849 :
850 112 : for (int i = 0; i < server_count; i++) {
851 105 : const char *uid = server_uids[i];
852 :
853 : /* Binary search in sorted local UNREAD index */
854 105 : int found = 0;
855 105 : int lo = 0, hi = local_count - 1;
856 422 : while (lo <= hi) {
857 422 : int mid = lo + (hi - lo) / 2;
858 422 : int cmp = strcmp(local_uids[mid], uid);
859 422 : if (cmp == 0) { found = 1; break; }
860 317 : if (cmp < 0) lo = mid + 1; else hi = mid - 1;
861 : }
862 105 : if (found) continue; /* already in local UNREAD index */
863 :
864 0 : if (local_hdr_exists("", uid)) {
865 : /* Cached but missing UNREAD label — patch the .hdr */
866 0 : const char *add[] = {"UNREAD"};
867 0 : local_hdr_update_labels("", uid, add, 1, NULL, 0);
868 0 : label_idx_add("UNREAD", uid);
869 0 : updated++;
870 : } else {
871 : /* Not in local cache — download with current labels (includes UNREAD) */
872 0 : if (store_fetched_message(gc, uid, rules) == 0)
873 0 : fetched++;
874 : }
875 : }
876 :
877 7 : mail_rules_free(rules);
878 7 : free(server_uids);
879 7 : free(local_uids);
880 :
881 7 : if (updated > 0 || fetched > 0) {
882 0 : logger_log(LOG_INFO,
883 : "gmail_sync: recover_unread: patched=%d fetched=%d",
884 : updated, fetched);
885 0 : if (updated > 0)
886 0 : fprintf(stderr, " Recovered UNREAD flag for %d cached messages.\n",
887 : updated);
888 0 : if (fetched > 0)
889 0 : fprintf(stderr, " Downloaded %d new unread messages.\n", fetched);
890 : }
891 : }
892 :
893 : /* ── Incremental Sync ─────────────────────────────────────────────── */
894 :
895 13 : int gmail_sync_incremental(GmailClient *gc) {
896 13 : char *history_id = local_gmail_history_load();
897 13 : if (!history_id) {
898 2 : logger_log(LOG_INFO, "gmail_sync: no historyId, need full sync");
899 2 : return -2;
900 : }
901 :
902 11 : logger_log(LOG_INFO, "gmail_sync: incremental from historyId %s", history_id);
903 :
904 11 : char *resp = gmail_get_history(gc, history_id);
905 11 : free(history_id);
906 :
907 11 : if (!resp) {
908 3 : fprintf(stderr, " Incremental: History API returned error/404 (historyId expired or network issue).\n");
909 3 : logger_log(LOG_WARN, "gmail_sync: history expired or error");
910 3 : return -2; /* Signal: need full sync */
911 : }
912 :
913 8 : MailRules *inc_rules = mail_rules_load(local_store_account_name());
914 8 : struct history_ctx hc = { .gc = gc, .rules = inc_rules, .added = 0, .deleted = 0, .label_changes = 0 };
915 :
916 : /* Process each history record.
917 : * The response is {"history": [{...}, ...], "historyId": "..."}.
918 : * Each record may contain messagesAdded, messagesDeleted,
919 : * labelsAdded, labelsRemoved arrays.
920 : * process_history_record dispatches to the individual handlers. */
921 8 : json_foreach_object(resp, "history", process_history_record, &hc);
922 :
923 : /* Save updated historyId */
924 8 : RAII_STRING char *new_history_id = json_get_string(resp, "historyId");
925 8 : if (new_history_id)
926 8 : local_gmail_history_save(new_history_id);
927 :
928 8 : free(resp);
929 :
930 : /* Refresh label name mapping if any label events occurred */
931 8 : if (hc.label_changes > 0) {
932 1 : char **lbl_names = NULL, **lbl_ids = NULL;
933 1 : int lbl_count = 0;
934 1 : if (gmail_list_labels(gc, &lbl_names, &lbl_ids, &lbl_count) == 0) {
935 1 : local_gmail_label_names_save(lbl_ids, lbl_names, lbl_count);
936 4 : for (int i = 0; i < lbl_count; i++) { free(lbl_names[i]); free(lbl_ids[i]); }
937 1 : free(lbl_names);
938 1 : free(lbl_ids);
939 : }
940 : }
941 :
942 : /* Ensure no archived message is marked unread (repair existing data too) */
943 8 : gmail_sync_repair_archive_flags();
944 :
945 : /* Refresh UNREAD state from the server to correct any label changes that
946 : * were missed while the history-parsing bug was present. */
947 8 : recover_unread_labels(gc);
948 :
949 8 : mail_rules_free(inc_rules);
950 :
951 : /* Report what actually changed — "up to date" alone would hide the
952 : * messages this run downloaded. */
953 8 : if (hc.added || hc.deleted || hc.label_changes)
954 2 : fprintf(stderr, " Incremental sync: %d new, %d deleted, %d label change(s).\n",
955 : hc.added, hc.deleted, hc.label_changes);
956 : else
957 6 : fprintf(stderr, " Incremental sync: up to date.\n");
958 :
959 8 : logger_log(LOG_INFO, "gmail_sync: incremental done — added=%d deleted=%d labels=%d",
960 : hc.added, hc.deleted, hc.label_changes);
961 8 : return 0;
962 : }
963 :
964 : /* ── Auto Sync (public entry point) ───────────────────────────────── */
965 :
966 : /**
967 : * Smart sync flow:
968 : *
969 : * 1. If pending_fetch.tsv is non-empty, download those first (resuming an
970 : * interrupted previous sync or initial download).
971 : * 2. If the local store was already complete (no pending at start) AND a
972 : * valid historyId exists, use the fast incremental path (1–2 API calls).
973 : * 3. Otherwise, run reconcile (full UID listing) to discover any missing
974 : * messages, then download them.
975 : *
976 : * This ensures:
977 : * - First run or expired historyId: O(N) reconcile → O(missing) downloads.
978 : * - Subsequent runs on a mature store: O(1) incremental (no listing at all).
979 : * - Interrupted downloads: resume from pending_fetch.tsv without re-listing.
980 : */
981 69 : int gmail_sync(GmailClient *gc) {
982 : /* Step 1: check readiness before downloading anything */
983 69 : int had_pending = local_pending_fetch_count() > 0;
984 :
985 : /* Step 2: drain any queued downloads from a previous (possibly interrupted) sync */
986 69 : if (had_pending)
987 3 : gmail_sync_fetch_pending(gc);
988 :
989 : /* Step 3: try fast incremental path if we have a saved historyId.
990 : * We do this regardless of whether there were pending downloads —
991 : * draining pending_fetch.tsv already brought the local store up to the
992 : * reconcile snapshot; incremental then catches anything that arrived
993 : * on the server after that snapshot. */
994 69 : char *history_id = local_gmail_history_load();
995 69 : int have_history = (history_id != NULL);
996 69 : free(history_id);
997 :
998 69 : if (have_history) {
999 9 : fprintf(stderr, " Incremental sync (historyId present)...\n");
1000 9 : int rc = gmail_sync_incremental(gc);
1001 9 : if (rc == 0)
1002 7 : return 0; /* fast path — done; the callee reported the outcome */
1003 2 : if (rc != -2) return rc; /* unexpected error */
1004 2 : fprintf(stderr, " Incremental sync: historyId expired — falling back to full reconcile.\n");
1005 2 : logger_log(LOG_INFO, "gmail_sync: historyId expired, falling back to reconcile");
1006 : } else {
1007 60 : fprintf(stderr, " No saved historyId — full reconcile needed.\n");
1008 : }
1009 :
1010 : /* Step 4: reconcile (discover what is missing) */
1011 62 : int queued = gmail_sync_reconcile(gc);
1012 62 : if (queued < 0) return -1;
1013 :
1014 : /* Step 5: download what reconcile found */
1015 62 : if (queued > 0)
1016 52 : gmail_sync_fetch_pending(gc);
1017 :
1018 : /* Step 6: rebuild label indexes from .hdr files.
1019 : * Necessary when all messages were already cached (queued == 0) but
1020 : * label .idx files were deleted or are missing (e.g. manual deletion
1021 : * or upgrade from an older version). */
1022 : {
1023 62 : char (*all_uids)[17] = NULL;
1024 62 : int all_count = 0;
1025 62 : if (local_hdr_list_all_uids("", &all_uids, &all_count) == 0 && all_count > 0)
1026 61 : rebuild_label_indexes((const char (*)[17])all_uids, all_count);
1027 62 : free(all_uids);
1028 : }
1029 :
1030 62 : return 0;
1031 : }
|
