This commit is contained in:
2026-01-22 15:27:37 +01:00
commit 7212eb6f7a
35 changed files with 1462 additions and 0 deletions

5
backend/.env.example Normal file
View File

@@ -0,0 +1,5 @@
NODE_ENV=development
PORT=8000
DATABASE_URL=postgresql://mailcleaner:mailcleaner@localhost:5432/mailcleaner
REDIS_URL=redis://localhost:6379
JWT_SECRET=change-me-super-secret

14
backend/Dockerfile Normal file
View File

@@ -0,0 +1,14 @@
FROM node:20-slim
WORKDIR /app
COPY package.json package-lock.json* ./
RUN npm install
COPY tsconfig.json ./
COPY prisma ./prisma
COPY src ./src
EXPOSE 8000
CMD ["npm", "run", "dev"]

39
backend/package.json Normal file
View File

@@ -0,0 +1,39 @@
{
"name": "simple-mail-cleaner-backend",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"dev": "node --watch --loader ts-node/esm src/main.ts",
"worker:dev": "node --watch --loader ts-node/esm src/worker.ts",
"build": "tsc -p tsconfig.json",
"start": "node dist/main.js",
"start:worker": "node dist/worker.js",
"prisma:generate": "prisma generate",
"prisma:migrate": "prisma migrate dev"
},
"dependencies": {
"@fastify/cors": "^9.0.1",
"@fastify/helmet": "^12.1.0",
"@fastify/jwt": "^9.0.2",
"@fastify/swagger": "^9.3.0",
"@fastify/swagger-ui": "^4.2.0",
"@prisma/client": "^5.22.0",
"argon2": "^0.41.1",
"bullmq": "^5.48.1",
"fastify": "^4.28.1",
"fastify-plugin": "^4.5.1",
"imapflow": "^1.0.180",
"ioredis": "^5.5.0",
"mailparser": "^3.7.1",
"pino": "^9.5.0",
"pino-pretty": "^10.3.1",
"zod": "^3.24.2"
},
"devDependencies": {
"@types/node": "^22.10.7",
"prisma": "^5.22.0",
"ts-node": "^10.9.2",
"typescript": "^5.7.3"
}
}

View File

@@ -0,0 +1,198 @@
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
}
enum MailProvider {
GMAIL
GMX
WEBDE
}
enum JobStatus {
QUEUED
RUNNING
SUCCEEDED
FAILED
CANCELED
}
enum RuleActionType {
MOVE
DELETE
ARCHIVE
LABEL
}
enum RuleConditionType {
HEADER
SUBJECT
FROM
LIST_UNSUBSCRIBE
LIST_ID
}
model Tenant {
id String @id @default(cuid())
name String
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
users User[]
mailboxAccounts MailboxAccount[]
rules Rule[]
jobs CleanupJob[]
}
model User {
id String @id @default(cuid())
tenantId String
email String @unique
password String
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
tenant Tenant @relation(fields: [tenantId], references: [id])
}
model MailboxAccount {
id String @id @default(cuid())
tenantId String
email String
provider MailProvider
imapHost String
imapPort Int
imapTLS Boolean
smtpHost String?
smtpPort Int?
smtpTLS Boolean?
oauthToken String?
appPassword String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
tenant Tenant @relation(fields: [tenantId], references: [id])
folders MailboxFolder[]
jobs CleanupJob[]
@@index([tenantId])
}
model MailboxFolder {
id String @id @default(cuid())
mailboxAccountId String
name String
remoteId String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
mailboxAccount MailboxAccount @relation(fields: [mailboxAccountId], references: [id])
mailItems MailItem[]
@@index([mailboxAccountId])
}
model MailItem {
id String @id @default(cuid())
folderId String
messageId String
subject String?
from String?
receivedAt DateTime?
listId String?
listUnsubscribe String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
folder MailboxFolder @relation(fields: [folderId], references: [id])
@@index([folderId])
@@index([messageId])
}
model Rule {
id String @id @default(cuid())
tenantId String
name String
enabled Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
tenant Tenant @relation(fields: [tenantId], references: [id])
conditions RuleCondition[]
actions RuleAction[]
@@index([tenantId])
}
model RuleCondition {
id String @id @default(cuid())
ruleId String
type RuleConditionType
value String
rule Rule @relation(fields: [ruleId], references: [id])
@@index([ruleId])
}
model RuleAction {
id String @id @default(cuid())
ruleId String
type RuleActionType
target String?
rule Rule @relation(fields: [ruleId], references: [id])
@@index([ruleId])
}
model CleanupJob {
id String @id @default(cuid())
tenantId String
mailboxAccountId String
status JobStatus @default(QUEUED)
startedAt DateTime?
finishedAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
tenant Tenant @relation(fields: [tenantId], references: [id])
mailboxAccount MailboxAccount @relation(fields: [mailboxAccountId], references: [id])
unsubscribeAttempts UnsubscribeAttempt[]
events CleanupJobEvent[]
@@index([tenantId])
@@index([mailboxAccountId])
}
model UnsubscribeAttempt {
id String @id @default(cuid())
jobId String
mailItemId String?
method String
target String
status String
createdAt DateTime @default(now())
job CleanupJob @relation(fields: [jobId], references: [id])
@@index([jobId])
}
model CleanupJobEvent {
id String @id @default(cuid())
jobId String
level String
message String
progress Int?
createdAt DateTime @default(now())
job CleanupJob @relation(fields: [jobId], references: [id])
@@index([jobId])
}

View File

@@ -0,0 +1,12 @@
import fp from "fastify-plugin";
import { FastifyInstance } from "fastify";
export default fp(async function authPlugin(app: FastifyInstance) {
app.decorate("authenticate", async (request, reply) => {
try {
await request.jwtVerify();
} catch (err) {
reply.code(401).send({ message: "Unauthorized" });
}
});
});

View File

@@ -0,0 +1,63 @@
import { FastifyInstance } from "fastify";
import argon2 from "argon2";
import { z } from "zod";
import { prisma } from "../db.js";
const registerSchema = z.object({
tenantName: z.string().min(2),
email: z.string().email(),
password: z.string().min(10)
});
const loginSchema = z.object({
email: z.string().email(),
password: z.string().min(1)
});
export async function authRoutes(app: FastifyInstance) {
app.post("/register", async (request, reply) => {
const input = registerSchema.parse(request.body);
const existing = await prisma.user.findUnique({ where: { email: input.email } });
if (existing) {
return reply.code(409).send({ message: "Email already registered" });
}
const hashed = await argon2.hash(input.password);
const tenant = await prisma.tenant.create({
data: { name: input.tenantName }
});
const user = await prisma.user.create({
data: {
tenantId: tenant.id,
email: input.email,
password: hashed
}
});
const token = app.jwt.sign({ sub: user.id, tenantId: user.tenantId });
return { token, user: { id: user.id, email: user.email, tenantId: user.tenantId } };
});
app.post("/login", async (request, reply) => {
const input = loginSchema.parse(request.body);
const user = await prisma.user.findUnique({ where: { email: input.email } });
if (!user) {
return reply.code(401).send({ message: "Invalid credentials" });
}
const valid = await argon2.verify(user.password, input.password);
if (!valid) {
return reply.code(401).send({ message: "Invalid credentials" });
}
const token = app.jwt.sign({ sub: user.id, tenantId: user.tenantId });
return { token, user: { id: user.id, email: user.email, tenantId: user.tenantId } };
});
app.post("/logout", async () => ({ success: true }));
}

19
backend/src/config.ts Normal file
View File

@@ -0,0 +1,19 @@
import { z } from "zod";
const envSchema = z.object({
NODE_ENV: z.string().default("development"),
PORT: z.coerce.number().default(8000),
DATABASE_URL: z.string().url(),
REDIS_URL: z.string().url(),
JWT_SECRET: z.string().min(12)
});
export type AppConfig = z.infer<typeof envSchema>;
export const config = envSchema.parse({
NODE_ENV: process.env.NODE_ENV,
PORT: process.env.PORT,
DATABASE_URL: process.env.DATABASE_URL,
REDIS_URL: process.env.REDIS_URL,
JWT_SECRET: process.env.JWT_SECRET
});

5
backend/src/db.ts Normal file
View File

@@ -0,0 +1,5 @@
import { PrismaClient } from "@prisma/client";
export const prisma = new PrismaClient({
log: ["error", "warn"]
});

View File

@@ -0,0 +1,5 @@
import { FastifyInstance } from "fastify";
export async function healthRoutes(app: FastifyInstance) {
app.get("/", async () => ({ status: "ok" }));
}

View File

@@ -0,0 +1,50 @@
import { prisma } from "../db.js";
import { logJobEvent } from "../queue/jobEvents.js";
import { createImapClient, fetchHeaders, listMailboxes } from "./imap.js";
import { detectNewsletter } from "./newsletter.js";
export const runCleanup = async (cleanupJobId: string, mailboxAccountId: string) => {
const account = await prisma.mailboxAccount.findUnique({ where: { id: mailboxAccountId } });
if (!account) {
throw new Error("Mailbox account not found");
}
await logJobEvent(cleanupJobId, "info", `Connecting to ${account.email}`);
const client = createImapClient(account);
await client.connect();
try {
const mailboxes = await listMailboxes(client);
await logJobEvent(cleanupJobId, "info", `Found ${mailboxes.length} mailboxes`, 10);
const targetMailbox = mailboxes.find((box) => /inbox/i.test(box.path))?.path ?? "INBOX";
await logJobEvent(cleanupJobId, "info", `Scanning ${targetMailbox}`, 20);
const headers = await fetchHeaders(client, targetMailbox, 300);
let newsletterCount = 0;
for (const msg of headers) {
const result = detectNewsletter({
headers: msg.headers,
subject: msg.subject,
from: msg.from
});
if (result.isNewsletter) {
newsletterCount += 1;
}
}
await logJobEvent(cleanupJobId, "info", `Detected ${newsletterCount} newsletter candidates`, 60);
await logJobEvent(
cleanupJobId,
"info",
"Routing, unsubscribe, and deletion steps will be executed in the next phase",
80
);
} finally {
await client.logout().catch(() => undefined);
}
};

54
backend/src/mail/imap.ts Normal file
View File

@@ -0,0 +1,54 @@
import { ImapFlow } from "imapflow";
import { simpleParser } from "mailparser";
import { MailboxAccount } from "@prisma/client";
export const createImapClient = (account: MailboxAccount) => {
return new ImapFlow({
host: account.imapHost,
port: account.imapPort,
secure: account.imapTLS,
auth: account.oauthToken
? { user: account.email, accessToken: account.oauthToken }
: { user: account.email, pass: account.appPassword ?? "" }
});
};
export const listMailboxes = async (client: ImapFlow) => {
const mailboxes = [] as { name: string; path: string }[];
for await (const mailbox of client.list()) {
mailboxes.push({ name: mailbox.name, path: mailbox.path });
}
return mailboxes;
};
export const fetchHeaders = async (client: ImapFlow, mailbox: string, limit = 500) => {
await client.mailboxOpen(mailbox);
const messages = [] as {
uid: number;
subject?: string;
from?: string;
headers: Map<string, string>;
}[];
const search = await client.search({ all: true });
const slice = search.slice(-limit);
for await (const msg of client.fetch(slice, { envelope: true, source: true })) {
const parsed = await simpleParser(msg.source ?? "");
const headers = new Map<string, string>();
for (const [key, value] of parsed.headers) {
headers.set(key.toLowerCase(), Array.isArray(value) ? value.join(",") : String(value));
}
messages.push({
uid: msg.uid,
subject: parsed.subject,
from: parsed.from?.text,
headers
});
}
return messages;
};

View File

@@ -0,0 +1,45 @@
const headerIncludes = (headers: Map<string, string>, key: string) =>
headers.has(key.toLowerCase());
const headerValue = (headers: Map<string, string>, key: string) =>
headers.get(key.toLowerCase()) ?? "";
const containsAny = (value: string, tokens: string[]) =>
tokens.some((token) => value.includes(token));
export const detectNewsletter = (params: {
headers: Map<string, string>;
subject?: string | null;
from?: string | null;
}) => {
const subject = (params.subject ?? "").toLowerCase();
const from = (params.from ?? "").toLowerCase();
const headers = params.headers;
const hasListUnsubscribe = headerIncludes(headers, "list-unsubscribe");
const hasListId = headerIncludes(headers, "list-id");
const precedence = headerValue(headers, "precedence").toLowerCase();
const bulkHeader = headerValue(headers, "x-precedence").toLowerCase();
const headerHints = containsAny(precedence, ["bulk", "list"]) ||
containsAny(bulkHeader, ["bulk", "list"]) ||
headerIncludes(headers, "list-unsubscribe-post");
const subjectHints = containsAny(subject, ["newsletter", "unsubscribe", "update", "news", "digest"]);
const fromHints = containsAny(from, ["newsletter", "no-reply", "noreply", "news", "updates"]);
const score = [hasListUnsubscribe, hasListId, headerHints, subjectHints, fromHints].filter(Boolean).length;
return {
isNewsletter: score >= 2,
score,
signals: {
hasListUnsubscribe,
hasListId,
headerHints,
subjectHints,
fromHints
}
};
};

View File

@@ -0,0 +1,35 @@
import { MailProvider } from "@prisma/client";
export const providerDefaults: Record<MailProvider, {
imapHost: string;
imapPort: number;
imapTLS: boolean;
smtpHost: string;
smtpPort: number;
smtpTLS: boolean;
}> = {
GMAIL: {
imapHost: "imap.gmail.com",
imapPort: 993,
imapTLS: true,
smtpHost: "smtp.gmail.com",
smtpPort: 587,
smtpTLS: true
},
GMX: {
imapHost: "imap.gmx.net",
imapPort: 993,
imapTLS: true,
smtpHost: "smtp.gmx.net",
smtpPort: 587,
smtpTLS: true
},
WEBDE: {
imapHost: "imap.web.de",
imapPort: 993,
imapTLS: true,
smtpHost: "smtp.web.de",
smtpPort: 587,
smtpTLS: true
}
};

View File

@@ -0,0 +1,81 @@
import { FastifyInstance } from "fastify";
import { z } from "zod";
import { prisma } from "../db.js";
import { providerDefaults } from "./providers.js";
import { queueCleanupJob } from "../queue/queue.js";
const createAccountSchema = z.object({
email: z.string().email(),
provider: z.enum(["GMAIL", "GMX", "WEBDE"]),
imapHost: z.string().optional(),
imapPort: z.number().optional(),
imapTLS: z.boolean().optional(),
smtpHost: z.string().optional(),
smtpPort: z.number().optional(),
smtpTLS: z.boolean().optional(),
oauthToken: z.string().optional(),
appPassword: z.string().optional()
});
const cleanupSchema = z.object({
mailboxAccountId: z.string()
});
export async function mailRoutes(app: FastifyInstance) {
app.addHook("preHandler", app.authenticate);
app.get("/accounts", async (request) => {
const accounts = await prisma.mailboxAccount.findMany({
where: { tenantId: request.user.tenantId }
});
return { accounts };
});
app.post("/accounts", async (request, reply) => {
const input = createAccountSchema.parse(request.body);
const defaults = providerDefaults[input.provider];
const account = await prisma.mailboxAccount.create({
data: {
tenantId: request.user.tenantId,
email: input.email,
provider: input.provider,
imapHost: input.imapHost ?? defaults.imapHost,
imapPort: input.imapPort ?? defaults.imapPort,
imapTLS: input.imapTLS ?? defaults.imapTLS,
smtpHost: input.smtpHost ?? defaults.smtpHost,
smtpPort: input.smtpPort ?? defaults.smtpPort,
smtpTLS: input.smtpTLS ?? defaults.smtpTLS,
oauthToken: input.oauthToken,
appPassword: input.appPassword
}
});
return reply.code(201).send({ account });
});
app.get("/mailboxes", async () => ({ mailboxes: [] }));
app.post("/cleanup", async (request, reply) => {
const input = cleanupSchema.parse(request.body);
const account = await prisma.mailboxAccount.findFirst({
where: { id: input.mailboxAccountId, tenantId: request.user.tenantId }
});
if (!account) {
return reply.code(404).send({ message: "Mailbox account not found" });
}
const job = await prisma.cleanupJob.create({
data: {
tenantId: request.user.tenantId,
mailboxAccountId: account.id
}
});
await queueCleanupJob(job.id, account.id);
return reply.code(202).send({ jobId: job.id });
});
}

51
backend/src/main.ts Normal file
View File

@@ -0,0 +1,51 @@
import Fastify from "fastify";
import cors from "@fastify/cors";
import helmet from "@fastify/helmet";
import jwt from "@fastify/jwt";
import swagger from "@fastify/swagger";
import swaggerUi from "@fastify/swagger-ui";
import { config } from "./config.js";
import authPlugin from "./auth/plugin.js";
import { healthRoutes } from "./health/routes.js";
import { authRoutes } from "./auth/routes.js";
import { tenantRoutes } from "./tenant/routes.js";
import { mailRoutes } from "./mail/routes.js";
import { queueRoutes } from "./queue/routes.js";
const app = Fastify({
logger: {
transport: {
target: "pino-pretty",
options: { colorize: true }
}
}
});
await app.register(cors, { origin: true });
await app.register(helmet);
await app.register(jwt, { secret: config.JWT_SECRET });
await app.register(authPlugin);
await app.register(swagger, {
openapi: {
info: { title: "Simple Mail Cleaner API", version: "0.1.0" }
}
});
await app.register(swaggerUi, { routePrefix: "/docs" });
await app.register(healthRoutes, { prefix: "/health" });
await app.register(authRoutes, { prefix: "/auth" });
await app.register(tenantRoutes, { prefix: "/tenants" });
await app.register(mailRoutes, { prefix: "/mail" });
await app.register(queueRoutes, { prefix: "/jobs" });
const start = async () => {
try {
await app.listen({ port: config.PORT, host: "0.0.0.0" });
} catch (err) {
app.log.error(err);
process.exit(1);
}
};
await start();

View File

@@ -0,0 +1,12 @@
import { prisma } from "../db.js";
export const logJobEvent = async (jobId: string, level: string, message: string, progress?: number) => {
await prisma.cleanupJobEvent.create({
data: {
jobId,
level,
message,
progress
}
});
};

View File

@@ -0,0 +1,25 @@
import { Queue } from "bullmq";
import IORedis from "ioredis";
import { config } from "../config.js";
let cleanupQueue: Queue | null = null;
const getConnection = () => new IORedis(config.REDIS_URL, { maxRetriesPerRequest: null });
export const getCleanupQueue = () => {
if (!cleanupQueue) {
cleanupQueue = new Queue("cleanup", {
connection: getConnection()
});
}
return cleanupQueue;
};
export const queueCleanupJob = async (cleanupJobId: string, mailboxAccountId: string) => {
const queue = getCleanupQueue();
await queue.add(
"cleanup",
{ cleanupJobId, mailboxAccountId },
{ jobId: cleanupJobId }
);
};

View File

@@ -0,0 +1,48 @@
import { FastifyInstance } from "fastify";
import { prisma } from "../db.js";
export async function queueRoutes(app: FastifyInstance) {
app.addHook("preHandler", app.authenticate);
app.get("/", async (request) => {
const jobs = await prisma.cleanupJob.findMany({
where: { tenantId: request.user.tenantId },
orderBy: { createdAt: "desc" }
});
return { jobs };
});
app.get("/:id", async (request, reply) => {
const params = request.params as { id: string };
const job = await prisma.cleanupJob.findFirst({
where: { id: params.id, tenantId: request.user.tenantId }
});
if (!job) {
return reply.code(404).send({ message: "Job not found" });
}
return { job };
});
app.get("/:id/events", async (request, reply) => {
const params = request.params as { id: string };
const job = await prisma.cleanupJob.findFirst({
where: { id: params.id, tenantId: request.user.tenantId }
});
if (!job) {
return reply.code(404).send({ message: "Job not found" });
}
const events = await prisma.cleanupJobEvent.findMany({
where: { jobId: job.id },
orderBy: { createdAt: "asc" }
});
return { events };
});
}

View File

@@ -0,0 +1,18 @@
import { FastifyInstance } from "fastify";
import { prisma } from "../db.js";
export async function tenantRoutes(app: FastifyInstance) {
app.addHook("preHandler", app.authenticate);
app.get("/me", async (request) => {
const user = await prisma.user.findUnique({
where: { id: request.user.sub },
include: { tenant: true }
});
return {
user: user ? { id: user.id, email: user.email } : null,
tenant: user?.tenant ? { id: user.tenant.id, name: user.tenant.name } : null
};
});
}

16
backend/src/types.d.ts vendored Normal file
View File

@@ -0,0 +1,16 @@
import "@fastify/jwt";
import type { FastifyReply, FastifyRequest } from "fastify";
import "fastify";
declare module "@fastify/jwt" {
interface FastifyJWT {
payload: { sub: string; tenantId: string };
user: { sub: string; tenantId: string };
}
}
declare module "fastify" {
interface FastifyInstance {
authenticate: (request: FastifyRequest, reply: FastifyReply) => Promise<void>;
}
}

48
backend/src/worker.ts Normal file
View File

@@ -0,0 +1,48 @@
import { Worker } from "bullmq";
import IORedis from "ioredis";
import { prisma } from "./db.js";
import { config } from "./config.js";
import { runCleanup } from "./mail/cleanup.js";
import { logJobEvent } from "./queue/jobEvents.js";
const connection = new IORedis(config.REDIS_URL, { maxRetriesPerRequest: null });
const worker = new Worker(
"cleanup",
async (job) => {
const { cleanupJobId, mailboxAccountId } = job.data as { cleanupJobId: string; mailboxAccountId: string };
await prisma.cleanupJob.update({
where: { id: cleanupJobId },
data: { status: "RUNNING", startedAt: new Date() }
});
await logJobEvent(cleanupJobId, "info", "Cleanup started", 5);
await runCleanup(cleanupJobId, mailboxAccountId);
await prisma.cleanupJob.update({
where: { id: cleanupJobId },
data: { status: "SUCCEEDED", finishedAt: new Date() }
});
await logJobEvent(cleanupJobId, "info", "Cleanup finished", 100);
return { ok: true };
},
{ connection }
);
worker.on("failed", async (job, err) => {
if (!job) return;
const cleanupJobId = job.data?.cleanupJobId as string | undefined;
if (!cleanupJobId) return;
await prisma.cleanupJob.update({
where: { id: cleanupJobId },
data: { status: "FAILED", finishedAt: new Date() }
});
await logJobEvent(cleanupJobId, "error", `Job failed: ${err.message}`);
process.stderr.write(`[worker] job ${cleanupJobId} failed: ${err.message}\n`);
});
process.stdout.write("[worker] cleanup worker ready\n");

15
backend/tsconfig.json Normal file
View File

@@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"outDir": "dist",
"rootDir": "src",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
"skipLibCheck": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist"]
}