|
|
|
|
@@ -177,6 +177,19 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type GmailClient = Awaited<ReturnType<typeof gmailClientForAccount>>["gmail"];
|
|
|
|
|
type RuleAction = { type: string; target?: string | null };
|
|
|
|
|
type ActionLogItem = { type: string; target?: string | null; status: string; error?: string };
|
|
|
|
|
type ImapBatchItem = {
|
|
|
|
|
uid: number;
|
|
|
|
|
candidateId: string;
|
|
|
|
|
actions: RuleAction[];
|
|
|
|
|
actionLogItems: ActionLogItem[];
|
|
|
|
|
actionLog: ActionLogItem[];
|
|
|
|
|
hasDelete: boolean;
|
|
|
|
|
moveAction?: RuleAction | null;
|
|
|
|
|
markAction?: RuleAction | null;
|
|
|
|
|
};
|
|
|
|
|
type ImapBatchQueue = { items: ImapBatchItem[] };
|
|
|
|
|
|
|
|
|
|
const processMessage = async (msg: {
|
|
|
|
|
uid: number;
|
|
|
|
|
@@ -186,7 +199,7 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
headers: Map<string, string>;
|
|
|
|
|
gmailMessageId?: string;
|
|
|
|
|
mailbox?: string;
|
|
|
|
|
}, gmailContext?: { gmail: GmailClient; resolveLabelId: (label: string) => Promise<string> }) => {
|
|
|
|
|
}, gmailContext?: { gmail: GmailClient; resolveLabelId: (label: string) => Promise<string> }, imapBatch?: ImapBatchQueue) => {
|
|
|
|
|
const ctx = {
|
|
|
|
|
headers: msg.headers,
|
|
|
|
|
subject: msg.subject ?? "",
|
|
|
|
|
@@ -235,6 +248,225 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const actionLog: ActionLogItem[] = [];
|
|
|
|
|
let deferredImap = false;
|
|
|
|
|
|
|
|
|
|
const applyActions = async (actionsToApply: RuleAction[], allowBatch: boolean) => {
|
|
|
|
|
if (!actionsToApply.length) return;
|
|
|
|
|
const deferImapActions = Boolean(allowBatch && imapBatch && !job.dryRun && account.provider !== "GMAIL");
|
|
|
|
|
if (deferImapActions) {
|
|
|
|
|
deferredImap = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (job.dryRun) {
|
|
|
|
|
for (const action of actionsToApply) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `DRY RUN: ${action.type} ${action.target ?? ""}`);
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "dry-run" });
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (account.provider === "GMAIL" && msg.gmailMessageId && gmailContext) {
|
|
|
|
|
const actionStart = Date.now();
|
|
|
|
|
actionAttempts += 1;
|
|
|
|
|
const actionLogItems: ActionLogItem[] = actionsToApply.map((item) => ({
|
|
|
|
|
type: item.type,
|
|
|
|
|
target: item.target ?? null,
|
|
|
|
|
status: "pending" as const,
|
|
|
|
|
error: undefined as string | undefined
|
|
|
|
|
}));
|
|
|
|
|
const hasDelete = actionsToApply.some((item) => item.type === "DELETE");
|
|
|
|
|
try {
|
|
|
|
|
if (hasDelete) {
|
|
|
|
|
await gmailContext.gmail.users.messages.delete({ userId: "me", id: msg.gmailMessageId });
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Gmail action DELETE applied");
|
|
|
|
|
gmailDeleteCount += 1;
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.type === "DELETE") {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
} else {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
const addLabelIds = new Set<string>();
|
|
|
|
|
const removeLabelIds = new Set<string>();
|
|
|
|
|
for (const item of actionsToApply) {
|
|
|
|
|
if ((item.type === "MOVE" || item.type === "LABEL") && item.target) {
|
|
|
|
|
const labelId = await gmailContext.resolveLabelId(item.target);
|
|
|
|
|
addLabelIds.add(labelId);
|
|
|
|
|
if (item.type === "MOVE") {
|
|
|
|
|
removeLabelIds.add("INBOX");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "ARCHIVE") {
|
|
|
|
|
removeLabelIds.add("INBOX");
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MARK_READ") {
|
|
|
|
|
removeLabelIds.add("UNREAD");
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MARK_UNREAD") {
|
|
|
|
|
addLabelIds.add("UNREAD");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (addLabelIds.size === 0 && removeLabelIds.size === 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Gmail action skipped: no label changes");
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
await gmailContext.gmail.users.messages.modify({
|
|
|
|
|
userId: "me",
|
|
|
|
|
id: msg.gmailMessageId,
|
|
|
|
|
requestBody: {
|
|
|
|
|
addLabelIds: Array.from(addLabelIds),
|
|
|
|
|
removeLabelIds: Array.from(removeLabelIds)
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail action applied: ${actionsToApply.map((a) => a.type).join(", ")}`);
|
|
|
|
|
gmailModifyCount += 1;
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `Gmail action failed: ${message}`);
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.status === "pending") {
|
|
|
|
|
item.status = "failed";
|
|
|
|
|
item.error = message;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
routingSeconds += (Date.now() - actionStart) / 1000;
|
|
|
|
|
}
|
|
|
|
|
actionLog.push(...actionLogItems);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (deferImapActions && imapBatch) {
|
|
|
|
|
const actionLogItems: ActionLogItem[] = actionsToApply.map((item) => ({
|
|
|
|
|
type: item.type,
|
|
|
|
|
target: item.target ?? null,
|
|
|
|
|
status: "pending",
|
|
|
|
|
error: undefined
|
|
|
|
|
}));
|
|
|
|
|
const moveCandidates = actionsToApply.filter(
|
|
|
|
|
(item) => (item.type === "MOVE" || item.type === "ARCHIVE" || item.type === "LABEL") && item.target
|
|
|
|
|
);
|
|
|
|
|
const moveAction = moveCandidates.length ? moveCandidates[moveCandidates.length - 1] : null;
|
|
|
|
|
const markCandidates = actionsToApply.filter((item) => item.type === "MARK_READ" || item.type === "MARK_UNREAD");
|
|
|
|
|
const markAction = markCandidates.length ? markCandidates[markCandidates.length - 1] : null;
|
|
|
|
|
const hasDelete = actionsToApply.some((item) => item.type === "DELETE");
|
|
|
|
|
actionLog.push(...actionLogItems);
|
|
|
|
|
imapBatch.items.push({
|
|
|
|
|
uid: msg.uid,
|
|
|
|
|
candidateId: candidate.id,
|
|
|
|
|
actions: actionsToApply,
|
|
|
|
|
actionLogItems,
|
|
|
|
|
actionLog,
|
|
|
|
|
hasDelete,
|
|
|
|
|
moveAction,
|
|
|
|
|
markAction
|
|
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!imapClient) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Skipping IMAP action: no IMAP client");
|
|
|
|
|
for (const action of actionsToApply) {
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "skipped" });
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const actionStart = Date.now();
|
|
|
|
|
actionAttempts += 1;
|
|
|
|
|
const actionLogItems = actionsToApply.map((item) => ({
|
|
|
|
|
type: item.type,
|
|
|
|
|
target: item.target ?? null,
|
|
|
|
|
status: "pending" as const,
|
|
|
|
|
error: undefined as string | undefined
|
|
|
|
|
}));
|
|
|
|
|
const hasDelete = actionsToApply.some((item) => item.type === "DELETE");
|
|
|
|
|
const moveCandidates = actionsToApply.filter(
|
|
|
|
|
(item) => (item.type === "MOVE" || item.type === "ARCHIVE" || item.type === "LABEL") && item.target
|
|
|
|
|
);
|
|
|
|
|
const moveAction = moveCandidates.length ? moveCandidates[moveCandidates.length - 1] : null;
|
|
|
|
|
const markCandidates = actionsToApply.filter((item) => item.type === "MARK_READ" || item.type === "MARK_UNREAD");
|
|
|
|
|
const markAction = markCandidates.length ? markCandidates[markCandidates.length - 1] : null;
|
|
|
|
|
try {
|
|
|
|
|
if (hasDelete) {
|
|
|
|
|
await imapClient.messageDelete(msg.uid);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Deleted message ${msg.uid}`);
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.type === "DELETE") {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
} else {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (markAction?.type === "MARK_READ") {
|
|
|
|
|
await imapClient.messageFlagsAdd(msg.uid, ["\\Seen"]);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Marked message ${msg.uid} as read`);
|
|
|
|
|
}
|
|
|
|
|
if (markAction?.type === "MARK_UNREAD") {
|
|
|
|
|
await imapClient.messageFlagsRemove(msg.uid, ["\\Seen"]);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Marked message ${msg.uid} as unread`);
|
|
|
|
|
}
|
|
|
|
|
if (moveAction?.target) {
|
|
|
|
|
if (!imapMailboxCache.has(moveAction.target)) {
|
|
|
|
|
await imapClient.mailboxCreate(moveAction.target).catch(() => undefined);
|
|
|
|
|
imapMailboxCache.add(moveAction.target);
|
|
|
|
|
}
|
|
|
|
|
await imapClient.messageMove(msg.uid, moveAction.target);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Moved message ${msg.uid} to ${moveAction.target}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (markAction && item.type === markAction.type) {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MARK_READ" || item.type === "MARK_UNREAD") {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (moveAction && item.type === moveAction.type && item.target === moveAction.target) {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MOVE" || item.type === "ARCHIVE" || item.type === "LABEL") {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "DELETE") {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP action failed: ${message}`);
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.status === "pending") {
|
|
|
|
|
item.status = "failed";
|
|
|
|
|
item.error = message;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
routingSeconds += (Date.now() - actionStart) / 1000;
|
|
|
|
|
}
|
|
|
|
|
actionLog.push(...actionLogItems);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const preRoutingCtx = { ...ctx, unsubscribeStatus: null, newsletterScore: result.score };
|
|
|
|
|
const preActions = job.routingEnabled ? matchRules(rules, preRoutingCtx, "PRE_UNSUBSCRIBE") : [];
|
|
|
|
|
await applyActions(preActions, false);
|
|
|
|
|
|
|
|
|
|
let unsubscribeStatus = job.unsubscribeEnabled ? "pending" : "disabled";
|
|
|
|
|
let unsubscribeMessage: string | null = null;
|
|
|
|
|
let unsubscribeDetails: Record<string, any> | null = null;
|
|
|
|
|
@@ -338,141 +570,23 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const routingCtx = { ...ctx, unsubscribeStatus, newsletterScore: result.score };
|
|
|
|
|
const actions = job.routingEnabled ? matchRules(rules, routingCtx) : [];
|
|
|
|
|
const actionLog: { type: string; target?: string | null; status: string; error?: string }[] = [];
|
|
|
|
|
if (actions.length > 0) {
|
|
|
|
|
for (const action of actions) {
|
|
|
|
|
if (job.dryRun) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `DRY RUN: ${action.type} ${action.target ?? ""}`);
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "dry-run" });
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (account.provider === "GMAIL" && msg.gmailMessageId && gmailContext) {
|
|
|
|
|
const actionStart = Date.now();
|
|
|
|
|
actionAttempts += 1;
|
|
|
|
|
const actionLogItems = actions.map((item) => ({
|
|
|
|
|
type: item.type,
|
|
|
|
|
target: item.target ?? null,
|
|
|
|
|
status: "pending" as const,
|
|
|
|
|
error: undefined as string | undefined
|
|
|
|
|
}));
|
|
|
|
|
const hasDelete = actions.some((item) => item.type === "DELETE");
|
|
|
|
|
try {
|
|
|
|
|
if (hasDelete) {
|
|
|
|
|
await gmailContext.gmail.users.messages.delete({ userId: "me", id: msg.gmailMessageId });
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Gmail action DELETE applied");
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.type === "DELETE") {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
} else {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
const addLabelIds = new Set<string>();
|
|
|
|
|
const removeLabelIds = new Set<string>();
|
|
|
|
|
for (const item of actions) {
|
|
|
|
|
if ((item.type === "MOVE" || item.type === "LABEL") && item.target) {
|
|
|
|
|
const labelId = await gmailContext.resolveLabelId(item.target);
|
|
|
|
|
addLabelIds.add(labelId);
|
|
|
|
|
if (item.type === "MOVE") {
|
|
|
|
|
removeLabelIds.add("INBOX");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "ARCHIVE") {
|
|
|
|
|
removeLabelIds.add("INBOX");
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MARK_READ") {
|
|
|
|
|
removeLabelIds.add("UNREAD");
|
|
|
|
|
}
|
|
|
|
|
if (item.type === "MARK_UNREAD") {
|
|
|
|
|
addLabelIds.add("UNREAD");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (addLabelIds.size === 0 && removeLabelIds.size === 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Gmail action skipped: no label changes");
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
item.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
await gmailContext.gmail.users.messages.modify({
|
|
|
|
|
userId: "me",
|
|
|
|
|
id: msg.gmailMessageId,
|
|
|
|
|
requestBody: {
|
|
|
|
|
addLabelIds: Array.from(addLabelIds),
|
|
|
|
|
removeLabelIds: Array.from(removeLabelIds)
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail action applied: ${actions.map((a) => a.type).join(", ")}`);
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
item.status = "applied";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `Gmail action failed: ${message}`);
|
|
|
|
|
for (const item of actionLogItems) {
|
|
|
|
|
if (item.status === "pending") {
|
|
|
|
|
item.status = "failed";
|
|
|
|
|
item.error = message;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
routingSeconds += (Date.now() - actionStart) / 1000;
|
|
|
|
|
}
|
|
|
|
|
actionLog.push(...actionLogItems);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!imapClient) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", "Skipping IMAP action: no IMAP client");
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "skipped" });
|
|
|
|
|
} else {
|
|
|
|
|
const actionStart = Date.now();
|
|
|
|
|
actionAttempts += 1;
|
|
|
|
|
try {
|
|
|
|
|
if ((action.type === "MOVE" || action.type === "ARCHIVE" || action.type === "LABEL") && action.target) {
|
|
|
|
|
await imapClient.mailboxCreate(action.target).catch(() => undefined);
|
|
|
|
|
await imapClient.messageMove(msg.uid, action.target);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Moved message ${msg.uid} to ${action.target}`);
|
|
|
|
|
}
|
|
|
|
|
if (action.type === "DELETE") {
|
|
|
|
|
await imapClient.messageDelete(msg.uid);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Deleted message ${msg.uid}`);
|
|
|
|
|
}
|
|
|
|
|
if (action.type === "MARK_READ") {
|
|
|
|
|
await imapClient.messageFlagsAdd(msg.uid, ["\\Seen"]);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Marked message ${msg.uid} as read`);
|
|
|
|
|
}
|
|
|
|
|
if (action.type === "MARK_UNREAD") {
|
|
|
|
|
await imapClient.messageFlagsRemove(msg.uid, ["\\Seen"]);
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Marked message ${msg.uid} as unread`);
|
|
|
|
|
}
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "applied" });
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP action ${action.type} failed: ${message}`);
|
|
|
|
|
actionLog.push({ type: action.type, target: action.target ?? null, status: "failed", error: message });
|
|
|
|
|
} finally {
|
|
|
|
|
routingSeconds += (Date.now() - actionStart) / 1000;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const postRoutingCtx = { ...ctx, unsubscribeStatus, newsletterScore: result.score };
|
|
|
|
|
const postActions = job.routingEnabled ? matchRules(rules, postRoutingCtx, "POST_UNSUBSCRIBE") : [];
|
|
|
|
|
await applyActions(postActions, true);
|
|
|
|
|
|
|
|
|
|
if (actionLog.length || unsubscribeStatus !== "pending" || unsubscribeTarget) {
|
|
|
|
|
const updateData: Record<string, any> = {
|
|
|
|
|
unsubscribeStatus,
|
|
|
|
|
unsubscribeMessage,
|
|
|
|
|
unsubscribeTarget,
|
|
|
|
|
unsubscribeDetails
|
|
|
|
|
};
|
|
|
|
|
if (actionLog.length && !deferredImap) {
|
|
|
|
|
updateData.actions = actionLog;
|
|
|
|
|
}
|
|
|
|
|
await prisma.cleanupJobCandidate.update({
|
|
|
|
|
where: { id: candidate.id },
|
|
|
|
|
data: {
|
|
|
|
|
actions: actionLog.length ? actionLog : undefined,
|
|
|
|
|
unsubscribeStatus,
|
|
|
|
|
unsubscribeMessage,
|
|
|
|
|
unsubscribeTarget,
|
|
|
|
|
unsubscribeDetails
|
|
|
|
|
}
|
|
|
|
|
data: updateData
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -498,6 +612,10 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
let unsubscribeAttempts = 0;
|
|
|
|
|
let actionAttempts = 0;
|
|
|
|
|
const imapMailboxCache = new Set<string>();
|
|
|
|
|
let gmailModifyCount = 0;
|
|
|
|
|
let gmailDeleteCount = 0;
|
|
|
|
|
let gmailModifyLogged = 0;
|
|
|
|
|
let gmailDeleteLogged = 0;
|
|
|
|
|
|
|
|
|
|
if (isGmail && hasGmailOAuth) {
|
|
|
|
|
const { gmail } = await gmailClientForAccount(account);
|
|
|
|
|
@@ -657,10 +775,28 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
buildProgressMessage("Processed", processed, total, processingStart),
|
|
|
|
|
progress
|
|
|
|
|
);
|
|
|
|
|
const modifyDelta = gmailModifyCount - gmailModifyLogged;
|
|
|
|
|
if (modifyDelta > 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail batch: MODIFY ${modifyDelta}`);
|
|
|
|
|
gmailModifyLogged = gmailModifyCount;
|
|
|
|
|
}
|
|
|
|
|
const deleteDelta = gmailDeleteCount - gmailDeleteLogged;
|
|
|
|
|
if (deleteDelta > 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail batch: DELETE ${deleteDelta}`);
|
|
|
|
|
gmailDeleteLogged = gmailDeleteCount;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Detected ${newsletterCount} newsletter candidates`, 92);
|
|
|
|
|
const finalModifyDelta = gmailModifyCount - gmailModifyLogged;
|
|
|
|
|
if (finalModifyDelta > 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail batch: MODIFY ${finalModifyDelta}`);
|
|
|
|
|
}
|
|
|
|
|
const finalDeleteDelta = gmailDeleteCount - gmailDeleteLogged;
|
|
|
|
|
if (finalDeleteDelta > 0) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Gmail batch: DELETE ${finalDeleteDelta}`);
|
|
|
|
|
}
|
|
|
|
|
processingSeconds = Math.max(1, Math.round((Date.now() - processingStart) / 1000));
|
|
|
|
|
await prisma.cleanupJob.update({
|
|
|
|
|
where: { id: cleanupJobId },
|
|
|
|
|
@@ -747,6 +883,157 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `Processing ${total} IMAP messages`, 35);
|
|
|
|
|
const processingStart = Date.now();
|
|
|
|
|
let newsletterCount = 0;
|
|
|
|
|
const imapBatchQueue: ImapBatchQueue = { items: [] };
|
|
|
|
|
const flushImapBatch = async () => {
|
|
|
|
|
if (!imapBatchQueue.items.length || !imapClient) return;
|
|
|
|
|
const deleteItems: ImapBatchItem[] = [];
|
|
|
|
|
const markReadItems: ImapBatchItem[] = [];
|
|
|
|
|
const markUnreadItems: ImapBatchItem[] = [];
|
|
|
|
|
const moveItemsByTarget = new Map<string, ImapBatchItem[]>();
|
|
|
|
|
|
|
|
|
|
for (const item of imapBatchQueue.items) {
|
|
|
|
|
if (item.hasDelete) {
|
|
|
|
|
deleteItems.push(item);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (item.markAction?.type === "MARK_READ") {
|
|
|
|
|
markReadItems.push(item);
|
|
|
|
|
}
|
|
|
|
|
if (item.markAction?.type === "MARK_UNREAD") {
|
|
|
|
|
markUnreadItems.push(item);
|
|
|
|
|
}
|
|
|
|
|
if (item.moveAction?.target) {
|
|
|
|
|
const list = moveItemsByTarget.get(item.moveAction.target) ?? [];
|
|
|
|
|
list.push(item);
|
|
|
|
|
moveItemsByTarget.set(item.moveAction.target, list);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const errors: {
|
|
|
|
|
delete: string | null;
|
|
|
|
|
markRead: string | null;
|
|
|
|
|
markUnread: string | null;
|
|
|
|
|
move: Map<string, string | null>;
|
|
|
|
|
} = {
|
|
|
|
|
delete: null,
|
|
|
|
|
markRead: null,
|
|
|
|
|
markUnread: null,
|
|
|
|
|
move: new Map<string, string | null>()
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const actionStart = Date.now();
|
|
|
|
|
try {
|
|
|
|
|
if (deleteItems.length) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `IMAP batch: DELETE ${deleteItems.length}`);
|
|
|
|
|
try {
|
|
|
|
|
await imapClient.messageDelete(deleteItems.map((item) => item.uid));
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
errors.delete = message;
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP delete batch failed: ${message}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (markReadItems.length) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `IMAP batch: MARK_READ ${markReadItems.length}`);
|
|
|
|
|
try {
|
|
|
|
|
await imapClient.messageFlagsAdd(markReadItems.map((item) => item.uid), ["\\Seen"]);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
errors.markRead = message;
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP mark-read batch failed: ${message}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (markUnreadItems.length) {
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `IMAP batch: MARK_UNREAD ${markUnreadItems.length}`);
|
|
|
|
|
try {
|
|
|
|
|
await imapClient.messageFlagsRemove(markUnreadItems.map((item) => item.uid), ["\\Seen"]);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
|
|
|
errors.markUnread = message;
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP mark-unread batch failed: ${message}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for (const [target, items] of moveItemsByTarget.entries()) {
|
|
|
|
|
let moveError: string | null = null;
|
|
|
|
|
try {
|
|
|
|
|
if (!imapMailboxCache.has(target)) {
|
|
|
|
|
await imapClient.mailboxCreate(target).catch(() => undefined);
|
|
|
|
|
imapMailboxCache.add(target);
|
|
|
|
|
}
|
|
|
|
|
await logJobEvent(cleanupJobId, "info", `IMAP batch: MOVE ${items.length} -> ${target}`);
|
|
|
|
|
await imapClient.messageMove(items.map((item) => item.uid), target);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
moveError = err instanceof Error ? err.message : String(err);
|
|
|
|
|
await logJobEvent(cleanupJobId, "error", `IMAP move batch failed: ${moveError}`);
|
|
|
|
|
}
|
|
|
|
|
errors.move.set(target, moveError);
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
routingSeconds += (Date.now() - actionStart) / 1000;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const finalizeLog = (item: ImapBatchItem) => {
|
|
|
|
|
const deleteFailed = item.hasDelete && errors.delete;
|
|
|
|
|
const deleteSucceeded = item.hasDelete && !errors.delete;
|
|
|
|
|
const markAction = item.markAction;
|
|
|
|
|
const moveAction = item.moveAction;
|
|
|
|
|
const markFailed =
|
|
|
|
|
markAction?.type === "MARK_READ"
|
|
|
|
|
? errors.markRead
|
|
|
|
|
: markAction?.type === "MARK_UNREAD"
|
|
|
|
|
? errors.markUnread
|
|
|
|
|
: null;
|
|
|
|
|
const moveFailed = moveAction?.target ? errors.move.get(moveAction.target) ?? null : null;
|
|
|
|
|
|
|
|
|
|
for (const logItem of item.actionLogItems) {
|
|
|
|
|
if (item.hasDelete) {
|
|
|
|
|
if (logItem.type === "DELETE") {
|
|
|
|
|
logItem.status = deleteFailed ? "failed" : "applied";
|
|
|
|
|
if (deleteFailed) logItem.error = deleteFailed;
|
|
|
|
|
} else {
|
|
|
|
|
logItem.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (logItem.type === "MARK_READ" || logItem.type === "MARK_UNREAD") {
|
|
|
|
|
if (markAction && logItem.type === markAction.type) {
|
|
|
|
|
logItem.status = markFailed ? "failed" : "applied";
|
|
|
|
|
if (markFailed) logItem.error = markFailed;
|
|
|
|
|
} else {
|
|
|
|
|
logItem.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (logItem.type === "MOVE" || logItem.type === "ARCHIVE" || logItem.type === "LABEL") {
|
|
|
|
|
if (moveAction && logItem.type === moveAction.type && logItem.target === moveAction.target) {
|
|
|
|
|
logItem.status = moveFailed ? "failed" : "applied";
|
|
|
|
|
if (moveFailed) logItem.error = moveFailed ?? undefined;
|
|
|
|
|
} else {
|
|
|
|
|
logItem.status = "skipped";
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (logItem.type === "DELETE") {
|
|
|
|
|
logItem.status = deleteFailed ? "failed" : deleteSucceeded ? "applied" : "skipped";
|
|
|
|
|
if (deleteFailed) logItem.error = deleteFailed;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
for (const item of imapBatchQueue.items) {
|
|
|
|
|
finalizeLog(item);
|
|
|
|
|
await prisma.cleanupJobCandidate.update({
|
|
|
|
|
where: { id: item.candidateId },
|
|
|
|
|
data: { actions: item.actionLog }
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
actionAttempts += imapBatchQueue.items.length;
|
|
|
|
|
imapBatchQueue.items = [];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
for (let index = nextIndex; index < imapUids.length; index += checkpointEvery) {
|
|
|
|
|
const statusCheck = await prisma.cleanupJob.findUnique({ where: { id: cleanupJobId } });
|
|
|
|
|
@@ -766,10 +1053,11 @@ export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string)
|
|
|
|
|
processed += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const isNewsletter = await processMessage({ ...msg, mailbox: targetMailbox });
|
|
|
|
|
const isNewsletter = await processMessage({ ...msg, mailbox: targetMailbox }, undefined, imapBatchQueue);
|
|
|
|
|
if (isNewsletter) newsletterCount += 1;
|
|
|
|
|
processed += 1;
|
|
|
|
|
}
|
|
|
|
|
await flushImapBatch();
|
|
|
|
|
|
|
|
|
|
const nextProcessed = Math.min(total, index + batch.length);
|
|
|
|
|
await saveProgress(nextProcessed, total);
|
|
|
|
|
|