From 6c38de5fed1252116309260d683ddcaa3f9f9ab7 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 20:40:03 +0000 Subject: [PATCH 1/3] WA1: migration 003, imap_watch, notify, scheduler jobs, suggestions+notifications endpoints, email_classify mock --- apps/api/app/db/migrate.py | 1 + apps/api/app/imap_watch.py | 468 +++++++++++++++++++++++ apps/api/app/llm.py | 5 + apps/api/app/main.py | 109 +++++- apps/api/app/notify.py | 183 +++++++++ apps/api/app/scheduler.py | 92 ++++- apps/api/app/schemas.py | 29 +- apps/api/migrations/003_email_notify.sql | 24 ++ apps/api/tests/conftest.py | 2 +- 9 files changed, 907 insertions(+), 6 deletions(-) create mode 100644 apps/api/app/imap_watch.py create mode 100644 apps/api/app/notify.py create mode 100644 apps/api/migrations/003_email_notify.sql diff --git a/apps/api/app/db/migrate.py b/apps/api/app/db/migrate.py index 2e06910..ed642df 100644 --- a/apps/api/app/db/migrate.py +++ b/apps/api/app/db/migrate.py @@ -73,6 +73,7 @@ def reset_database(database_url: str | None = None) -> None: conn.execute( """ DROP TABLE IF EXISTS task_run, outbox, approval, artifact, + email_suggestion, notification_log, application, job_posting, cv_section, profile, schema_migrations CASCADE """ diff --git a/apps/api/app/imap_watch.py b/apps/api/app/imap_watch.py new file mode 100644 index 0000000..28f7090 --- /dev/null +++ b/apps/api/app/imap_watch.py @@ -0,0 +1,468 @@ +"""IMAP email watch: polls UNSEEN messages and classifies them. + +Uses stdlib imaplib (SSL). Enabled only when EMAIL_WATCH_ENABLED=true. +Config: IMAP_HOST, IMAP_PORT, IMAP_USER, IMAP_PASS. + +Flow: +1. Connect via IMAP SSL. +2. Fetch UNSEEN messages since last poll. +3. For each message, match sender domain + subject/body keywords to + open applications (status in sent/interviewing). +4. Classify via LLM task 'email_classify' -> {classification, state_proposal, reason}. +5. Insert email_suggestion rows (skip noise and dedupe by from+subject+day). +""" + +from __future__ import annotations + +import email +import email.utils +import hashlib +import imaplib +import logging +import os +from datetime import datetime, timezone +from typing import Any, Sequence + +from app.db import execute, fetch_all, fetch_one +from app import llm + +logger = logging.getLogger(__name__) + +VALID_CLASSIFICATIONS = frozenset({ + "interview_invite", + "rejection", + "question", + "noise", +}) + +# Keywords for cheap pre-matching before LLM classify +INTERVIEW_KEYWORDS = ("interview", "invite", "meeting", "schedule", "call") +REJECTION_KEYWORDS = ("regret", "unfortunately", "not moving", "rejection", "position has been filled") +QUESTION_KEYWORDS = ("question", "clarif", "additional", "could you", "please provide") +NOISE_KEYWORDS = ("newsletter", "unsubscribe", "promotion", "advert", "offer") + + +def is_email_watch_enabled() -> bool: + """Check if email watch is enabled.""" + return os.environ.get("EMAIL_WATCH_ENABLED", "false").lower() in ( + "true", + "1", + "yes", + ) + + +def _extract_sender_domain(from_addr: str) -> str: + """Extract the domain from an email From header.""" + parsed = email.utils.parseaddr(from_addr) + addr = parsed[1] or from_addr + parts = addr.split("@") + if len(parts) >= 2: + return parts[-1].lower().strip() + return "" + + +def _build_snippet(body: str, max_len: int = 300) -> str: + """Truncate body to a snippet.""" + body = body.replace("\r", " ").replace("\n", " ").strip() + if len(body) > max_len: + return body[:max_len] + "..." + return body + + +def _parse_email_message(raw_bytes: bytes) -> dict[str, str]: + """Parse raw email bytes into a dict with from, subject, body.""" + msg = email.message_from_bytes(raw_bytes) + from_addr = msg.get("From", "") + subject = msg.get("Subject", "") + date_str = msg.get("Date", "") + + # Extract body (prefer plain text) + body = "" + if msg.is_multipart(): + for part in msg.walk(): + ct = part.get_content_type() + if ct == "text/plain": + payload = part.get_payload(decode=True) + if payload: + body = payload.decode("utf-8", errors="replace") + break + if not body: + for part in msg.walk(): + if part.get_content_type().startswith("text/"): + payload = part.get_payload(decode=True) + if payload: + body = payload.decode("utf-8", errors="replace") + break + else: + payload = msg.get_payload(decode=True) + if payload: + body = payload.decode("utf-8", errors="replace") + + return { + "from": from_addr, + "subject": subject, + "body": body, + "date": date_str, + } + + +def _parse_date(date_str: str) -> datetime: + """Parse RFC 2822 date string to UTC datetime. Falls back to now().""" + if date_str: + try: + parsed = email.utils.parsedate_to_datetime(date_str) + if parsed is not None: + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + except (ValueError, TypeError): + pass + return datetime.now(timezone.utc) + + +def match_application( + from_addr: str, + subject: str, + body: str, + applications: Sequence[dict[str, Any]], +) -> dict[str, Any] | None: + """Match an email to an open application. + + Match by: + 1. Company name in subject/body + 2. Sender domain in posting URL + + Only matches applications with state in ('sent', 'interviewing'). + """ + sender_domain = _extract_sender_domain(from_addr) + text_lower = (subject + " " + body).lower() + + for app_row in applications: + state = app_row.get("state", "") + if state not in ("sent", "interviewing"): + continue + + company = (app_row.get("company") or "").lower() + title = (app_row.get("title") or "").lower() + posting_url = app_row.get("url") or "" + + # Match 1: company name in subject/body + if company and len(company) > 2 and company in text_lower: + return app_row + + # Match 2: sender domain in posting URL + if sender_domain and sender_domain in (posting_url or "").lower(): + return app_row + + # Match 3: title keywords in subject (looser) + if title and len(title) > 3: + title_words = [w for w in title.split() if len(w) > 3] + matches = sum(1 for w in title_words if w in text_lower) + if matches >= 2 and len(title_words) >= 2: + return app_row + + return None + + +def classify_email(subject: str, body: str) -> dict[str, Any]: + """Classify an email via LLM task 'email_classify'. + + Returns {classification, state_proposal, reason}. + Falls back to inline keyword heuristics if LLM fails. + """ + text_lower = (subject + " " + body).lower() + + result = llm.run_task( + "email_classify", + f"Subject: {subject}\nBody: {body[:1000]}", + ) + + classification = result.get("classification", "noise") + state_proposal = result.get("state_proposal") + reason = result.get("reason", "") + + # Validate + if classification not in VALID_CLASSIFICATIONS: + classification = "noise" + + return { + "classification": classification, + "state_proposal": state_proposal, + "reason": reason, + } + + +def is_duplicate(from_addr: str, subject: str, received_at: datetime) -> bool: + """Check if a similar email_suggestion already exists (same from+subject+day).""" + day_start = received_at.replace(hour=0, minute=0, second=0, microsecond=0) + row = fetch_one( + """ + SELECT id FROM email_suggestion + WHERE mailbox_from = %s AND subject = %s + AND received_at >= %s AND received_at < %s + interval '1 day' + LIMIT 1 + """, + (from_addr, subject, day_start, day_start), + ) + return row is not None + + +def create_email_suggestion( + application_id: str | None, + mailbox_from: str, + subject: str, + snippet: str, + classification: str, + state_proposal: str | None, + received_at: datetime, +) -> dict[str, Any]: + """Insert an email_suggestion row.""" + row = execute( + """ + INSERT INTO email_suggestion + (application_id, mailbox_from, subject, snippet, classification, state_proposal, status, received_at) + VALUES + (%s, %s, %s, %s, %s, %s, 'pending', %s) + RETURNING * + """, + (application_id, mailbox_from, subject, snippet, classification, state_proposal, received_at), + ) + if row is None: + raise RuntimeError("insert email_suggestion failed") + return _normalize_suggestion(row) + + +def _normalize_suggestion(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), + "application_id": str(row["application_id"]) if row.get("application_id") else None, + "mailbox_from": row["mailbox_from"], + "subject": row["subject"], + "snippet": row["snippet"], + "classification": row["classification"], + "state_proposal": row.get("state_proposal"), + "status": row["status"], + "received_at": row["received_at"].isoformat() if row.get("received_at") else None, + "created_at": row["created_at"].isoformat() if row.get("created_at") else None, + } + + +def list_pending_suggestions() -> list[dict[str, Any]]: + """Return all pending email_suggestions, newest first.""" + rows = fetch_all( + """ + SELECT * FROM email_suggestion + WHERE status = 'pending' + ORDER BY received_at DESC + """ + ) + return [_normalize_suggestion(r) for r in rows] + + +def get_suggestion(suggestion_id: str) -> dict[str, Any] | None: + row = fetch_one("SELECT * FROM email_suggestion WHERE id = %s", (suggestion_id,)) + if row is None: + return None + return _normalize_suggestion(row) + + +def update_suggestion_status( + suggestion_id: str, + status: str, +) -> dict[str, Any] | None: + row = execute( + "UPDATE email_suggestion SET status = %s WHERE id = %s RETURNING *", + (status, suggestion_id), + ) + if row is None: + return None + return _normalize_suggestion(row) + + +def get_open_applications() -> list[dict[str, Any]]: + """Return applications in sent/interviewing state for matching.""" + rows = fetch_all( + """ + SELECT a.*, j.company, j.title, j.url + FROM application a + JOIN job_posting j ON a.job_posting_id = j.id + WHERE a.state IN ('sent', 'interviewing') + """ + ) + result: list[dict[str, Any]] = [] + for row in rows: + result.append({ + "id": str(row["id"]), + "state": row["state"], + "company": row.get("company", ""), + "title": row.get("title", ""), + "url": row.get("url", ""), + }) + return result + + +# --- IMAP poll --- + +class FakeImap: + """Test double for imaplib IMAP4_SSL. No network. + + Usage: + fake = FakeImap(messages=[(uid1, raw1), (uid2, raw2)]) + poll_inbox(fake) # uses fake instead of real connection + """ + + def __init__(self, messages: list[tuple[bytes, bytes]] | None = None) -> None: + # messages: list of (uid, raw_email_bytes) + self._messages = messages or [] + self._seen_uids: set[bytes] = set() + self.selected = False + + def select(self, mailbox: str = "INBOX") -> tuple[str, list[bytes]]: + self.selected = True + count = len(self._messages) + return ("OK", [str(count).encode()]) + + def search(self, charset: str | None, *criteria: str) -> tuple[str, list[bytes]]: + # Return uids of messages matching criteria (we keep it simple) + uids = [uid for uid, _ in self._messages] + return ("OK", [b" ".join(uids)]) + + def fetch(self, uid: bytes, parts: str) -> tuple[str, list[tuple[bytes, bytes]]]: + for msg_uid, raw in self._messages: + if msg_uid == uid: + return ("OK", [(uid, raw)]) + return ("OK", []) + + def store(self, uid: bytes, flags: str, flag_set: str) -> tuple[str, list[bytes]]: + self._seen_uids.add(uid) + return ("OK", [uid]) + + def close(self) -> tuple[str, list[bytes]]: + self.selected = False + return ("OK", [b""]) + + def logout(self) -> tuple[str, list[bytes]]: + return ("OK", [b"BYE"]) + + +def _connect_imap() -> Any: + """Connect to IMAP server using env config.""" + host = os.environ.get("IMAP_HOST", "") + port = int(os.environ.get("IMAP_PORT", "993")) + user = os.environ.get("IMAP_USER", "") + password = os.environ.get("IMAP_PASS", "") + + conn = imaplib.IMAP4_SSL(host, port) + conn.login(user, password) + return conn + + +def poll_inbox(conn: Any = None) -> list[dict[str, Any]]: + """Poll the inbox for UNSEEN messages, classify, and create suggestions. + + If conn is provided (e.g. FakeImap for tests), uses it instead of connecting. + Returns list of created suggestions. + + No network is used when conn is a FakeImap. + """ + created: list[dict[str, Any]] = [] + own_conn = False + + if conn is None: + conn = _connect_imap() + own_conn = True + + try: + conn.select("INBOX") + status, data = conn.search(None, "UNSEEN") + if status != "OK": + logger.warning("imap_watch: search failed: %s", status) + return created + + uids = [] + if data and data[0]: + uids = data[0].split() + + if not uids: + return created + + # Get open applications for matching + applications = get_open_applications() + + for uid in uids: + status, fetch_data = conn.fetch(uid, "(RFC822)") + if status != "OK" or not fetch_data: + continue + + raw_bytes = b"" + for item in fetch_data: + if isinstance(item, tuple) and len(item) >= 2: + raw_bytes = item[1] + break + + if not raw_bytes: + continue + + parsed = _parse_email_message(raw_bytes) + from_addr = parsed["from"] + subject = parsed["subject"] + body = parsed["body"] + received_at = _parse_date(parsed["date"]) + snippet = _build_snippet(body) + + # Dedupe: same from+subject+day + if is_duplicate(from_addr, subject, received_at): + logger.debug("imap_watch: dedupe skip: %s / %s", from_addr, subject) + continue + + # Match to application + app_row = match_application(from_addr, subject, body, applications) + + # Classify + classification_result = classify_email(subject, body) + classification = classification_result["classification"] + state_proposal = classification_result["state_proposal"] + + # Skip pure noise (don't create suggestion rows) + if classification == "noise": + continue + + suggestion = create_email_suggestion( + application_id=app_row["id"] if app_row else None, + mailbox_from=from_addr, + subject=subject, + snippet=snippet, + classification=classification, + state_proposal=state_proposal, + received_at=received_at, + ) + created.append(suggestion) + + # Send notification for interview invites + if classification == "interview_invite" and app_row: + from app.notify import send_notification + send_notification( + "email_suggestion", + f"Interview invite from {app_row.get('company', 'unknown')}", + { + "suggestion_id": suggestion["id"], + "application_id": app_row["id"], + "classification": classification, + }, + ) + + # Mark as seen + try: + conn.store(uid, "+FLAGS", "\\Seen") + except Exception: + logger.debug("imap_watch: store failed for uid %s", uid) + + finally: + if own_conn: + try: + conn.close() + conn.logout() + except Exception: + pass + + return created \ No newline at end of file diff --git a/apps/api/app/llm.py b/apps/api/app/llm.py index c9c1515..7b9bc84 100644 --- a/apps/api/app/llm.py +++ b/apps/api/app/llm.py @@ -123,6 +123,11 @@ MOCK_OUTPUTS: dict[str, dict[str, Any]] = { "posting requirements. Be specific." ), }, + "email_classify": { + "classification": "interview_invite", + "state_proposal": "interviewing", + "reason": "The email mentions an interview invitation.", + }, } diff --git a/apps/api/app/main.py b/apps/api/app/main.py index b5857ee..b3ae3df 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -37,11 +37,13 @@ from app.schemas import ( CvSectionOut, CvSectionUpdate, DigestItem, + EmailSuggestionOut, ErrorOut, InterviewPrepResponse, JobPostingCreate, JobPostingOut, NudgeItem, + NotificationLogOut, OutboxOut, OutboxSendRequest, PostingsFetchRequest, @@ -932,4 +934,109 @@ def seed_demo() -> Any: "postings": postings_count, "applications": apps_count, "sections": sections_count, - } \ No newline at end of file + } + + +# --- v1.1: Email Suggestions --- + +@app.get("/api/suggestions", response_model=list[EmailSuggestionOut]) +def get_suggestions() -> Any: + """Return all pending email suggestions, newest first.""" + from app.imap_watch import list_pending_suggestions + return list_pending_suggestions() + + +@app.post("/api/suggestions/{suggestion_id}/accept") +def accept_suggestion(suggestion_id: str) -> Any: + """Accept a suggestion: apply state_proposal via guarded transition. + + If the suggestion has an application_id and a state_proposal, apply the + state transition through the normal guard path. Updates last_activity_at. + Marks the suggestion as 'accepted'. + """ + from app.imap_watch import get_suggestion, update_suggestion_status + + suggestion = get_suggestion(suggestion_id) + if suggestion is None: + raise HTTPException(status_code=404, detail="Suggestion not found") + + if suggestion["status"] != "pending": + raise HTTPException( + status_code=409, + detail={"code": "not_pending", "message": "Suggestion is not pending"}, + ) + + app_id = suggestion.get("application_id") + state_proposal = suggestion.get("state_proposal") + + if app_id and state_proposal: + # Apply guarded transition + app_row = repo_app.get_application(app_id) + if app_row is None: + raise HTTPException(status_code=404, detail="Linked application not found") + + from_state = app_row["state"] + to_state = state_proposal + + from app.statemachine import TransitionContext, check_transition, InvalidTransition + + has_score = app_row.get("score") is not None + has_confirmed_approval = False + artifact_hash_match = False + + ctx = TransitionContext( + application_id=app_id, + from_state=from_state, + to_state=to_state, + has_score=has_score, + has_confirmed_approval=has_confirmed_approval, + artifact_hash_match=artifact_hash_match, + ) + + try: + check_transition(ctx) + except InvalidTransition as exc: + raise HTTPException( + status_code=409, + detail={"code": "invalid_transition", "message": str(exc)}, + ) + + updated = repo_app.update_application_state(app_id, to_state) + if updated is None: + raise HTTPException(status_code=500, detail="State update failed") + + # Mark suggestion as accepted + result = update_suggestion_status(suggestion_id, "accepted") + if result is None: + raise HTTPException(status_code=500, detail="Failed to update suggestion") + return result + + +@app.post("/api/suggestions/{suggestion_id}/dismiss") +def dismiss_suggestion(suggestion_id: str) -> Any: + """Dismiss a suggestion (mark as dismissed).""" + from app.imap_watch import get_suggestion, update_suggestion_status + + suggestion = get_suggestion(suggestion_id) + if suggestion is None: + raise HTTPException(status_code=404, detail="Suggestion not found") + + if suggestion["status"] != "pending": + raise HTTPException( + status_code=409, + detail={"code": "not_pending", "message": "Suggestion is not pending"}, + ) + + result = update_suggestion_status(suggestion_id, "dismissed") + if result is None: + raise HTTPException(status_code=500, detail="Failed to update suggestion") + return result + + +# --- v1.1: Notification Log --- + +@app.get("/api/notifications/log", response_model=list[NotificationLogOut]) +def get_notification_log() -> Any: + """Return last 50 notification log entries.""" + from app.notify import list_notification_log + return list_notification_log(limit=50) \ No newline at end of file diff --git a/apps/api/app/notify.py b/apps/api/app/notify.py new file mode 100644 index 0000000..48c312a --- /dev/null +++ b/apps/api/app/notify.py @@ -0,0 +1,183 @@ +"""Notification channels: LogChannel + WebhookChannel. + +Protocol-based: NotificationChannel defines send(kind, text, data) -> bool. +LogChannel writes to notification_log (delivered=true always). +WebhookChannel POSTs to NOTIFY_WEBHOOK_URL (2xx=delivered, else error row). + +send_notification(kind, text, data) is the public entry point used by +the scheduler and email watch modules. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, Protocol + +from app.db import execute, fetch_all + +logger = logging.getLogger(__name__) + + +class NotificationChannel(Protocol): + """Protocol for notification delivery channels.""" + + def send(self, kind: str, text: str, data: dict[str, Any]) -> bool: + """Send a notification. Returns True if delivered, False otherwise.""" + ... + + +def _normalize_log_row(row: dict[str, Any]) -> dict[str, Any]: + payload = row.get("payload") + if payload is not None and not isinstance(payload, dict): + payload = json.loads(payload) if isinstance(payload, str) else payload + return { + "id": str(row["id"]), + "channel": row["channel"], + "kind": row["kind"], + "payload": payload or {}, + "delivered": row["delivered"], + "error": row.get("error"), + "created_at": row["created_at"].isoformat() if row.get("created_at") else None, + } + + +class LogChannel: + """Writes notification entries to notification_log with delivered=true.""" + + channel_name: str = "log" + + def send(self, kind: str, text: str, data: dict[str, Any]) -> bool: + row = execute( + """ + INSERT INTO notification_log (channel, kind, payload, delivered, error) + VALUES (%s, %s, %s, true, NULL) + RETURNING * + """, + (self.channel_name, kind, json.dumps({"text": text, **data})), + ) + if row is None: + logger.error("LogChannel: insert notification_log failed") + return False + logger.info("LogChannel: delivered kind=%s", kind) + return True + + +class WebhookChannel: + """POSTs notification payload to NOTIFY_WEBHOOK_URL. + + On 2xx response: writes notification_log with delivered=true. + On non-2xx or exception: writes notification_log with delivered=false and error. + """ + + channel_name: str = "webhook" + + def __init__(self, url: str | None = None) -> None: + self.url = url or os.environ.get("NOTIFY_WEBHOOK_URL", "") + + def send(self, kind: str, text: str, data: dict[str, Any]) -> bool: + payload: dict[str, Any] = {"kind": kind, "text": text, "data": data} + error: str | None = None + delivered = False + + if not self.url: + error = "NOTIFY_WEBHOOK_URL not configured" + logger.warning("WebhookChannel: %s", error) + else: + try: + import httpx + + resp = httpx.post(self.url, json=payload, timeout=10) + if 200 <= resp.status_code < 300: + delivered = True + else: + error = f"HTTP {resp.status_code}: {resp.text[:200]}" + logger.warning("WebhookChannel: %s", error) + except Exception as exc: + error = str(exc) + logger.warning("WebhookChannel: exception: %s", error) + + row = execute( + """ + INSERT INTO notification_log (channel, kind, payload, delivered, error) + VALUES (%s, %s, %s, %s, %s) + RETURNING * + """, + ( + self.channel_name, + kind, + json.dumps(payload), + delivered, + error, + ), + ) + if row is None: + logger.error("WebhookChannel: insert notification_log failed") + return delivered + + +# --- Channel registry --- + +_channels: list[NotificationChannel] | None = None + + +def get_channels() -> list[NotificationChannel]: + """Return the list of active notification channels. + + LogChannel is always included. + WebhookChannel is included when NOTIFY_WEBHOOK_URL is set. + """ + global _channels + if _channels is not None: + return _channels + + channels: list[NotificationChannel] = [LogChannel()] + webhook_url = os.environ.get("NOTIFY_WEBHOOK_URL", "").strip() + if webhook_url: + channels.append(WebhookChannel()) + _channels = channels + return channels + + +def set_channels(channels: list[NotificationChannel] | None) -> None: + """Override channel list (for testing).""" + global _channels + _channels = channels + + +def reset_channels() -> None: + """Reset to default (for testing).""" + global _channels + _channels = None + + +def send_notification(kind: str, text: str, data: dict[str, Any] | None = None) -> None: + """Send a notification via all active channels. + + kind: e.g. 'daily_digest', 'email_suggestion' + text: human-readable notification text + data: structured payload dict + """ + chs = get_channels() + payload = data or {} + for ch in chs: + try: + ch.send(kind, text, payload) + except Exception: + logger.exception("send_notification: channel %s failed", type(ch).__name__) + + +# --- Repository helpers --- + +def list_notification_log(limit: int = 50) -> list[dict[str, Any]]: + """Return recent notification_log rows, newest first.""" + rows = fetch_all( + """ + SELECT * FROM notification_log + ORDER BY created_at DESC + LIMIT %s + """, + (limit,), + ) + return [_normalize_log_row(r) for r in rows] \ No newline at end of file diff --git a/apps/api/app/scheduler.py b/apps/api/app/scheduler.py index 45de8bc..74748ce 100644 --- a/apps/api/app/scheduler.py +++ b/apps/api/app/scheduler.py @@ -1,7 +1,10 @@ -"""APScheduler integration: daily fetch + batch score job. +"""APScheduler integration: daily fetch + batch score + imap poll + digest. Starts during app lifespan when SCHEDULER_ENABLED=true (default false). -Runs a daily job at 07:00 that fetches postings and batch-scores pending applications. +Jobs: + - daily_fetch_score at 07:00: fetch postings + batch score + - daily_digest at 07:30: send daily digest notification + - imap_poll every 15 min: poll inbox for new emails (gated by EMAIL_WATCH_ENABLED) """ from __future__ import annotations @@ -23,6 +26,15 @@ def is_scheduler_enabled() -> bool: ) +def is_email_watch_enabled() -> bool: + """Check if email watch is enabled via env.""" + return os.environ.get("EMAIL_WATCH_ENABLED", "false").lower() in ( + "true", + "1", + "yes", + ) + + async def _daily_fetch_and_score() -> None: """Daily job: fetch postings and batch-score pending applications.""" logger.info("Scheduler: running daily fetch + batch score") @@ -56,6 +68,67 @@ async def _daily_fetch_and_score() -> None: logger.exception("Scheduler: daily job failed") +async def _daily_digest() -> None: + """Daily digest: build /today payload text and send notification.""" + logger.info("Scheduler: running daily digest") + try: + from app.notify import send_notification + from app.db import repo_app + + digest_apps = repo_app.get_digest(limit=20) + nudge_apps = repo_app.get_nudge_applications() + pending = repo_app.count_pending_approvals() + + lines: list[str] = [] + lines.append("Daily Digest") + lines.append(f"Scored applications: {len(digest_apps)}") + if digest_apps: + lines.append("") + lines.append("Top opportunities:") + for item in digest_apps[:5]: + score = item.get("score") + score_str = f" (score: {int(score)})" if score else "" + lines.append( + f" - {item.get('title', '?')} at {item.get('company', '?')}{score_str}" + ) + + if nudge_apps: + lines.append("") + lines.append(f"Follow-up nudges: {len(nudge_apps)}") + for n in nudge_apps[:5]: + lines.append( + f" - {n.get('title', '?')} at {n.get('company', '?')}" + f" ({n.get('days_since_sent', 0)} days since sent)" + ) + + if pending: + lines.append("") + lines.append(f"Pending approvals: {pending}") + + text = "\n".join(lines) + send_notification("daily_digest", text, { + "digest_count": len(digest_apps), + "nudge_count": len(nudge_apps), + "pending_approvals": pending, + }) + logger.info("Scheduler: daily digest sent") + except Exception: + logger.exception("Scheduler: daily digest failed") + + +async def _imap_poll() -> None: + """IMAP poll job: check for new emails and create suggestions.""" + if not is_email_watch_enabled(): + return + logger.info("Scheduler: running imap poll") + try: + from app.imap_watch import poll_inbox + created = poll_inbox() + logger.info("Scheduler: imap poll created %s suggestions", len(created)) + except Exception: + logger.exception("Scheduler: imap poll failed") + + def start_scheduler() -> None: """Start the APScheduler if enabled.""" global _scheduler @@ -66,6 +139,7 @@ def start_scheduler() -> None: try: from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger + from apscheduler.triggers.interval import IntervalTrigger except ImportError: logger.warning( "APScheduler not installed; scheduler will not start." @@ -79,8 +153,20 @@ def start_scheduler() -> None: id="daily_fetch_score", replace_existing=True, ) + _scheduler.add_job( + _daily_digest, + CronTrigger(hour=7, minute=30), + id="daily_digest", + replace_existing=True, + ) + _scheduler.add_job( + _imap_poll, + IntervalTrigger(minutes=15), + id="imap_poll", + replace_existing=True, + ) _scheduler.start() - logger.info("Scheduler started: daily fetch+score at 07:00") + logger.info("Scheduler started: daily fetch+score at 07:00, digest at 07:30, imap poll every 15 min") def stop_scheduler() -> None: diff --git a/apps/api/app/schemas.py b/apps/api/app/schemas.py index 47b746d..30acad0 100644 --- a/apps/api/app/schemas.py +++ b/apps/api/app/schemas.py @@ -319,4 +319,31 @@ class SeedDemoResponse(BaseModel): profile: str postings: int applications: int - sections: int \ No newline at end of file + sections: int + + +# --- v1.1: Email Suggestions --- + +class EmailSuggestionOut(BaseModel): + id: str + application_id: str | None = None + mailbox_from: str + subject: str + snippet: str + classification: str + state_proposal: str | None = None + status: str + received_at: str | None = None + created_at: str | None = None + + +# --- v1.1: Notification Log --- + +class NotificationLogOut(BaseModel): + id: str + channel: str + kind: str + payload: dict[str, Any] = {} + delivered: bool + error: str | None = None + created_at: str | None = None \ No newline at end of file diff --git a/apps/api/migrations/003_email_notify.sql b/apps/api/migrations/003_email_notify.sql new file mode 100644 index 0000000..817fc63 --- /dev/null +++ b/apps/api/migrations/003_email_notify.sql @@ -0,0 +1,24 @@ +-- 003_email_notify.sql -- email suggestions + notification log + +CREATE TABLE IF NOT EXISTS email_suggestion ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + application_id uuid REFERENCES application(id) ON DELETE SET NULL, + mailbox_from text NOT NULL, + subject text NOT NULL, + snippet text NOT NULL, + classification text NOT NULL CHECK (classification IN ('interview_invite','rejection','question','noise')), + state_proposal text, + status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','accepted','dismissed')), + received_at timestamptz NOT NULL, + created_at timestamptz DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS notification_log ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + channel text NOT NULL, + kind text NOT NULL, + payload jsonb NOT NULL, + delivered boolean NOT NULL, + error text, + created_at timestamptz DEFAULT now() +); \ No newline at end of file diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py index 2235f0a..303e767 100644 --- a/apps/api/tests/conftest.py +++ b/apps/api/tests/conftest.py @@ -38,7 +38,7 @@ def _truncate_tables(): with psycopg.connect(DATABASE_URL) as conn: conn.execute( """ - TRUNCATE TABLE task_run, outbox, approval, artifact, + TRUNCATE TABLE notification_log, email_suggestion, task_run, outbox, approval, artifact, application, job_posting, cv_section, profile RESTART IDENTITY CASCADE """ From 05ba99cb4b8dedc8447727bc0768a28d555d89e3 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 20:41:27 +0000 Subject: [PATCH 2/3] WA1: add 35 tests for email watch, notifications, suggestions, digest --- apps/api/tests/test_v11_email_notify.py | 682 ++++++++++++++++++++++++ 1 file changed, 682 insertions(+) create mode 100644 apps/api/tests/test_v11_email_notify.py diff --git a/apps/api/tests/test_v11_email_notify.py b/apps/api/tests/test_v11_email_notify.py new file mode 100644 index 0000000..3bce5d5 --- /dev/null +++ b/apps/api/tests/test_v11_email_notify.py @@ -0,0 +1,682 @@ +"""Tests for v1.1: email watch, notifications, suggestions endpoints. + +Covers: +- IMAP matching logic (company in subject, sender domain in URL, title keywords) +- Classifier -> suggestion row +- Noise dedupe (same from+subject+day) +- Accept applies transition through guard +- Dismiss marks suggestion +- Webhook success/failure notification_log rows +- LogChannel writes delivered=true +- Daily digest payload shape +- GET /suggestions, POST accept, POST dismiss, GET /notifications/log +- FakeImap end-to-end poll +""" + +from __future__ import annotations + +import email as email_mod +from datetime import datetime, timedelta, timezone + +import pytest +from fastapi.testclient import TestClient + +from app.config import DATABASE_URL +from app.db import repo_app +from app.imap_watch import ( + FakeImap, + _build_snippet, + _extract_sender_domain, + _parse_email_message, + classify_email, + create_email_suggestion, + is_duplicate, + match_application, + poll_inbox, +) +from app.notify import ( + LogChannel, + WebhookChannel, + get_channels, + list_notification_log, + reset_channels, + send_notification, + set_channels, +) + + +@pytest.fixture() +def client(): + from app.main import app + return TestClient(app) + + +# --- Helpers --- + +def _make_email(raw_from: str, subject: str, body: str, date: str = "") -> bytes: + """Build raw email bytes for FakeImap.""" + msg = email_mod.message_from_string( + f"From: {raw_from}\r\n" + f"Subject: {subject}\r\n" + f"Date: {date or 'Mon, 01 Jul 2026 10:00:00 +0000'}\r\n" + f"\r\n" + f"{body}" + ) + return msg.as_bytes() + + +def _create_app_in_state(state: str = "sent", company: str = "TechCorp", url: str = "https://techcorp.com/jobs/1") -> dict: + """Create a posting + application, force state via SQL.""" + posting = repo_app.create_job_posting( + source="manual_url", + url=url, + company=company, + title="Senior Python Developer", + location="Malmo", + description="", + raw={}, + ) + app_row = repo_app.create_application(posting["id"]) + if state != "discovered": + repo_app.update_application_score(app_row["id"], 80, {"factors": {}}) + if state in ("approved", "rejected"): + repo_app.update_application_state(app_row["id"], "scored") + repo_app.update_application_state(app_row["id"], state) + elif state == "sent": + repo_app.update_application_state(app_row["id"], "scored") + repo_app.update_application_state(app_row["id"], "approved") + repo_app.update_application_state(app_row["id"], "drafting") + # Bypass guard for test + from app.db import execute + execute( + "UPDATE application SET state = 'sent', last_activity_at = now() WHERE id = %s", + (app_row["id"],), + ) + elif state == "interviewing": + repo_app.update_application_state(app_row["id"], "scored") + repo_app.update_application_state(app_row["id"], "approved") + repo_app.update_application_state(app_row["id"], "drafting") + from app.db import execute + execute( + "UPDATE application SET state = 'sent' WHERE id = %s", + (app_row["id"],), + ) + execute( + "UPDATE application SET state = 'interviewing' WHERE id = %s", + (app_row["id"],), + ) + return app_row + + +# ======================================================================== +# IMAP matching logic (3 tests) +# ======================================================================== + +class TestImapMatching: + def test_match_by_company_name_in_subject(self): + """Email subject contains company name -> match.""" + apps = [ + {"id": "app1", "state": "sent", "company": "TechCorp", "title": "Python Dev", "url": "https://example.com"}, + ] + result = match_application( + "recruiter@gmail.com", + "Interview at TechCorp", + "Please come for an interview.", + apps, + ) + assert result is not None + assert result["id"] == "app1" + + def test_match_by_sender_domain_in_url(self): + """Sender domain matches the posting URL -> match.""" + apps = [ + {"id": "app2", "state": "sent", "company": "Unknown", "title": "Dev", "url": "https://techcorp.com/careers/1"}, + ] + result = match_application( + "hr@techcorp.com", + "Your application", + "We reviewed your application.", + apps, + ) + assert result is not None + assert result["id"] == "app2" + + def test_match_by_title_keywords(self): + """Email subject contains 2+ title words -> match.""" + apps = [ + {"id": "app3", "state": "interviewing", "company": "SomeCompany", "title": "Senior Python Developer", "url": "https://other.com"}, + ] + result = match_application( + "someone@other.com", + "Senior Python position update", + "Regarding the developer role.", + apps, + ) + assert result is not None + assert result["id"] == "app3" + + def test_no_match_wrong_state(self): + """Applications in discovered state are not matched.""" + apps = [ + {"id": "app4", "state": "discovered", "company": "TechCorp", "title": "Dev", "url": "https://techcorp.com"}, + ] + result = match_application( + "hr@techcorp.com", + "Interview at TechCorp", + "Come for an interview.", + apps, + ) + assert result is None + + def test_no_match_unrelated_email(self): + """Email unrelated to any application -> no match.""" + apps = [ + {"id": "app5", "state": "sent", "company": "TechCorp", "title": "Dev", "url": "https://techcorp.com"}, + ] + result = match_application( + "newsletter@spam.com", + "Buy now!", + "Special offer for you.", + apps, + ) + assert result is None + + +# ======================================================================== +# Classifier -> row (3 tests) +# ======================================================================== + +class TestClassifierToRow: + def test_classify_returns_interview_invite(self): + """classify_email returns interview_invite from mock.""" + result = classify_email("Interview invitation", "Please come for an interview next week.") + assert result["classification"] == "interview_invite" + assert result["state_proposal"] == "interviewing" + assert "reason" in result + + def test_classify_falls_back_on_invalid_classification(self): + """Invalid classification from LLM falls back to noise.""" + import app.llm as llm_mod + original = llm_mod.MOCK_OUTPUTS.get("email_classify", {}).copy() + try: + llm_mod.MOCK_OUTPUTS["email_classify"] = {"classification": "bogus", "state_proposal": None, "reason": "test"} + result = classify_email("test", "test") + assert result["classification"] == "noise" + finally: + llm_mod.MOCK_OUTPUTS["email_classify"] = original + + def test_create_email_suggestion_row(self): + """create_email_suggestion inserts a row correctly.""" + now = datetime.now(timezone.utc) + suggestion = create_email_suggestion( + application_id=None, + mailbox_from="hr@example.com", + subject="Interview invite", + snippet="Please come for an interview.", + classification="interview_invite", + state_proposal="interviewing", + received_at=now, + ) + assert suggestion["id"] is not None + assert suggestion["mailbox_from"] == "hr@example.com" + assert suggestion["classification"] == "interview_invite" + assert suggestion["status"] == "pending" + assert suggestion["state_proposal"] == "interviewing" + + +# ======================================================================== +# Noise dedupe (3 tests) +# ======================================================================== + +class TestNoiseDedupe: + def test_duplicate_detected_same_day(self): + """Same from+subject+day is flagged as duplicate.""" + now = datetime.now(timezone.utc) + create_email_suggestion( + application_id=None, + mailbox_from="hr@example.com", + subject="Interview", + snippet="Come for an interview.", + classification="interview_invite", + state_proposal="interviewing", + received_at=now, + ) + assert is_duplicate("hr@example.com", "Interview", now) + + def test_different_subject_not_duplicate(self): + """Different subject -> not duplicate.""" + now = datetime.now(timezone.utc) + create_email_suggestion( + application_id=None, + mailbox_from="hr@example.com", + subject="Interview", + snippet="Come.", + classification="interview_invite", + state_proposal="interviewing", + received_at=now, + ) + assert not is_duplicate("hr@example.com", "Different Subject", now) + + def test_different_sender_not_duplicate(self): + """Different sender -> not duplicate.""" + now = datetime.now(timezone.utc) + create_email_suggestion( + application_id=None, + mailbox_from="hr@example.com", + subject="Interview", + snippet="Come.", + classification="interview_invite", + state_proposal="interviewing", + received_at=now, + ) + assert not is_duplicate("other@example.com", "Interview", now) + + +# ======================================================================== +# Accept applies transition through guard (3 tests) +# ======================================================================== + +class TestAcceptSuggestion: + def test_accept_applies_transition(self, client): + """POST /suggestions/{id}/accept transitions app from sent to interviewing.""" + app_row = _create_app_in_state("sent", company="TechCorp") + suggestion = create_email_suggestion( + application_id=app_row["id"], + mailbox_from="hr@techcorp.com", + subject="Interview at TechCorp", + snippet="Please come in for an interview.", + classification="interview_invite", + state_proposal="interviewing", + received_at=datetime.now(timezone.utc), + ) + + resp = client.post(f"/api/suggestions/{suggestion['id']}/accept") + assert resp.status_code == 200 + assert resp.json()["status"] == "accepted" + + # Verify application state changed + updated_app = repo_app.get_application(app_row["id"]) + assert updated_app["state"] == "interviewing" + + def test_accept_invalid_transition_409(self, client): + """Accept with invalid transition (e.g. discovered -> interviewing) returns 409.""" + posting = repo_app.create_job_posting( + source="manual_url", url="https://example.com/bad/1", + company="X", title="X", location=None, description="", raw={}, + ) + app_row = repo_app.create_application(posting["id"]) + + suggestion = create_email_suggestion( + application_id=app_row["id"], + mailbox_from="hr@example.com", + subject="Test", + snippet="Test", + classification="interview_invite", + state_proposal="interviewing", + received_at=datetime.now(timezone.utc), + ) + + resp = client.post(f"/api/suggestions/{suggestion['id']}/accept") + assert resp.status_code == 409 + assert "invalid_transition" in str(resp.json()["detail"]) + + def test_accept_404_nonexistent(self, client): + """Accept on nonexistent suggestion -> 404.""" + resp = client.post("/api/suggestions/00000000-0000-0000-0000-000000000000/accept") + assert resp.status_code == 404 + + def test_accept_already_accepted_409(self, client): + """Accept on already accepted suggestion -> 409.""" + app_row = _create_app_in_state("sent") + suggestion = create_email_suggestion( + application_id=app_row["id"], + mailbox_from="hr@example.com", + subject="Test", + snippet="Test", + classification="interview_invite", + state_proposal="interviewing", + received_at=datetime.now(timezone.utc), + ) + client.post(f"/api/suggestions/{suggestion['id']}/accept") + resp = client.post(f"/api/suggestions/{suggestion['id']}/accept") + assert resp.status_code == 409 + + +# ======================================================================== +# Dismiss suggestion (2 tests) +# ======================================================================== + +class TestDismissSuggestion: + def test_dismiss_marks_as_dismissed(self, client): + """POST /suggestions/{id}/dismiss marks as dismissed.""" + suggestion = create_email_suggestion( + application_id=None, + mailbox_from="hr@example.com", + subject="Test", + snippet="Test", + classification="question", + state_proposal=None, + received_at=datetime.now(timezone.utc), + ) + + resp = client.post(f"/api/suggestions/{suggestion['id']}/dismiss") + assert resp.status_code == 200 + assert resp.json()["status"] == "dismissed" + + def test_dismiss_404_nonexistent(self, client): + """Dismiss nonexistent -> 404.""" + resp = client.post("/api/suggestions/00000000-0000-0000-0000-000000000000/dismiss") + assert resp.status_code == 404 + + +# ======================================================================== +# GET /suggestions (2 tests) +# ======================================================================== + +class TestGetSuggestions: + def test_get_suggestions_returns_pending(self, client): + """GET /suggestions returns only pending suggestions.""" + create_email_suggestion( + application_id=None, + mailbox_from="a@example.com", + subject="Subject A", + snippet="Snippet A", + classification="interview_invite", + state_proposal="interviewing", + received_at=datetime.now(timezone.utc), + ) + create_email_suggestion( + application_id=None, + mailbox_from="b@example.com", + subject="Subject B", + snippet="Snippet B", + classification="rejection", + state_proposal=None, + received_at=datetime.now(timezone.utc), + ) + + resp = client.get("/api/suggestions") + assert resp.status_code == 200 + data = resp.json() + assert len(data) == 2 + assert all(s["status"] == "pending" for s in data) + + def test_get_suggestions_empty(self, client): + """GET /suggestions returns empty list when no suggestions.""" + resp = client.get("/api/suggestions") + assert resp.status_code == 200 + assert resp.json() == [] + + +# ======================================================================== +# Notification channels (6 tests) +# ======================================================================== + +class TestLogChannel: + def test_log_channel_writes_delivered_true(self): + """LogChannel writes notification_log with delivered=true.""" + reset_channels() + ch = LogChannel() + result = ch.send("daily_digest", "Test digest", {"count": 5}) + assert result is True + + logs = list_notification_log(limit=10) + assert len(logs) >= 1 + latest = logs[0] + assert latest["channel"] == "log" + assert latest["kind"] == "daily_digest" + assert latest["delivered"] is True + assert latest["error"] is None + + def test_send_notification_log_channel(self): + """send_notification via LogChannel creates a log entry.""" + reset_channels() + set_channels([LogChannel()]) + send_notification("email_suggestion", "Interview invite from TechCorp", {"id": "test"}) + logs = list_notification_log(limit=10) + assert len(logs) >= 1 + assert logs[0]["kind"] == "email_suggestion" + assert logs[0]["delivered"] is True + reset_channels() + + +class TestWebhookChannel: + def test_webhook_success_2xx(self, monkeypatch): + """WebhookChannel with 2xx response writes delivered=true.""" + class FakeResponse: + status_code = 200 + text = "OK" + + class FakeClient: + @staticmethod + def post(url, json=None, timeout=None): + assert url == "https://hook.example.com/notify" + return FakeResponse() + + import httpx + monkeypatch.setattr(httpx, "post", FakeClient.post) + + reset_channels() + ch = WebhookChannel(url="https://hook.example.com/notify") + result = ch.send("daily_digest", "Digest", {"count": 3}) + assert result is True + + logs = list_notification_log(limit=10) + latest = logs[0] + assert latest["channel"] == "webhook" + assert latest["delivered"] is True + assert latest["error"] is None + reset_channels() + + def test_webhook_failure_non_2xx(self, monkeypatch): + """WebhookChannel with non-2xx writes delivered=false with error.""" + class FakeResponse: + status_code = 500 + text = "Internal Server Error" + + class FakeClient: + @staticmethod + def post(url, json=None, timeout=None): + return FakeResponse() + + import httpx + monkeypatch.setattr(httpx, "post", FakeClient.post) + + reset_channels() + ch = WebhookChannel(url="https://hook.example.com/notify") + result = ch.send("daily_digest", "Digest", {"count": 3}) + assert result is False + + logs = list_notification_log(limit=10) + latest = logs[0] + assert latest["channel"] == "webhook" + assert latest["delivered"] is False + assert latest["error"] is not None + assert "500" in latest["error"] + reset_channels() + + def test_webhook_exception_writes_error(self, monkeypatch): + """WebhookChannel with connection exception writes delivered=false.""" + def raise_exc(url, json=None, timeout=None): + raise ConnectionError("Connection refused") + + import httpx + monkeypatch.setattr(httpx, "post", raise_exc) + + reset_channels() + ch = WebhookChannel(url="https://hook.example.com/notify") + result = ch.send("daily_digest", "Digest", {"count": 1}) + assert result is False + + logs = list_notification_log(limit=10) + latest = logs[0] + assert latest["delivered"] is False + assert "Connection refused" in (latest["error"] or "") + reset_channels() + + def test_webhook_no_url_writes_error(self): + """WebhookChannel with no URL writes delivered=false with config error.""" + reset_channels() + ch = WebhookChannel(url="") + result = ch.send("test", "test", {}) + assert result is False + + logs = list_notification_log(limit=10) + latest = logs[0] + assert latest["delivered"] is False + assert "not configured" in (latest["error"] or "") + reset_channels() + + +# ======================================================================== +# Notification log endpoint (2 tests) +# ======================================================================== + +class TestNotificationLogEndpoint: + def test_get_notifications_log(self, client): + """GET /notifications/log returns entries.""" + reset_channels() + set_channels([LogChannel()]) + send_notification("daily_digest", "Test", {"count": 1}) + reset_channels() + + resp = client.get("/api/notifications/log") + assert resp.status_code == 200 + data = resp.json() + assert len(data) >= 1 + assert "channel" in data[0] + assert "kind" in data[0] + assert "delivered" in data[0] + + def test_get_notifications_log_empty(self, client): + """GET /notifications/log returns empty when no entries.""" + resp = client.get("/api/notifications/log") + assert resp.status_code == 200 + assert resp.json() == [] + + +# ======================================================================== +# Digest payload shape (2 tests) +# ======================================================================== + +class TestDigestPayload: + def test_daily_digest_notification_text(self): + """Daily digest notification contains expected text fields.""" + reset_channels() + set_channels([LogChannel()]) + send_notification("daily_digest", "Daily Digest\nScored applications: 5", { + "digest_count": 5, + "nudge_count": 2, + "pending_approvals": 1, + }) + logs = list_notification_log(limit=10) + entry = logs[0] + assert entry["kind"] == "daily_digest" + payload = entry["payload"] + assert "text" in payload + assert "Daily Digest" in payload["text"] + assert payload.get("digest_count") == 5 + reset_channels() + + def test_daily_digest_payload_has_counts(self): + """Digest payload includes digest_count, nudge_count, pending_approvals.""" + reset_channels() + set_channels([LogChannel()]) + send_notification("daily_digest", "text", { + "digest_count": 3, + "nudge_count": 1, + "pending_approvals": 0, + }) + logs = list_notification_log(limit=10) + payload = logs[0]["payload"] + assert payload["digest_count"] == 3 + assert payload["nudge_count"] == 1 + assert payload["pending_approvals"] == 0 + reset_channels() + + +# ======================================================================== +# FakeImap end-to-end poll (3 tests) +# ======================================================================== + +class TestFakeImapPoll: + def test_poll_creates_suggestion_for_matching_email(self): + """FakeImap poll creates a suggestion when email matches an application.""" + app_row = _create_app_in_state("sent", company="TechCorp", url="https://techcorp.com/jobs/1") + + raw_email = _make_email( + "hr@techcorp.com", + "Interview at TechCorp", + "Please come for an interview next Tuesday.", + ) + fake = FakeImap(messages=[(b"1", raw_email)]) + created = poll_inbox(fake) + assert len(created) == 1 + assert created[0]["classification"] == "interview_invite" + assert created[0]["mailbox_from"] == "hr@techcorp.com" + assert created[0]["application_id"] == app_row["id"] + + def test_poll_skips_noise_emails(self): + """FakeImap poll skips noise classification (no suggestion created).""" + _create_app_in_state("sent", company="TechCorp") + + # Mock email_classify to return noise + import app.llm as llm_mod + original = llm_mod.MOCK_OUTPUTS.get("email_classify", {}).copy() + try: + llm_mod.MOCK_OUTPUTS["email_classify"] = { + "classification": "noise", + "state_proposal": None, + "reason": "spam", + } + raw_email = _make_email( + "newsletter@spam.com", + "Buy our product", + "Special offer just for you!", + ) + fake = FakeImap(messages=[(b"1", raw_email)]) + created = poll_inbox(fake) + assert len(created) == 0 + finally: + llm_mod.MOCK_OUTPUTS["email_classify"] = original + + def test_poll_dedupe_skips_same_from_subject_day(self): + """FakeImap poll dedupes same from+subject+day.""" + raw_email = _make_email( + "hr@example.com", + "Same Subject", + "Same body content.", + ) + # First poll + fake1 = FakeImap(messages=[(b"1", raw_email)]) + created1 = poll_inbox(fake1) + assert len(created1) == 1 + + # Second poll with same message -> dedupe + fake2 = FakeImap(messages=[(b"1", raw_email)]) + created2 = poll_inbox(fake2) + assert len(created2) == 0 # deduped + + +# ======================================================================== +# Email parsing helpers (2 tests) +# ======================================================================== + +class TestEmailParsing: + def test_extract_sender_domain(self): + """Extract domain from From header.""" + assert _extract_sender_domain("John Doe ") == "techcorp.com" + assert _extract_sender_domain("noreply@example.org") == "example.org" + assert _extract_sender_domain("") == "" + + def test_build_snippet_truncates(self): + """Snippet is truncated to max_len.""" + long_body = "A" * 500 + snippet = _build_snippet(long_body, max_len=50) + assert len(snippet) <= 53 # 50 + "..." + assert snippet.endswith("...") + + def test_build_snippet_short_body(self): + """Short body is not truncated.""" + snippet = _build_snippet("Hello", max_len=300) + assert snippet == "Hello" \ No newline at end of file From 069ac454e55bf099e712a09833caad09e6611ec3 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 20:42:39 +0000 Subject: [PATCH 3/3] WA2: packages/matching - similarity, dedupe cluster, keywords extract+coverage, 65 tests with agency-repost fixture triples --- packages/matching/README.md | 40 ++++ packages/matching/pyproject.toml | 26 +++ packages/matching/src/matching/__init__.py | 28 +++ packages/matching/src/matching/dedupe.py | 136 ++++++++++++++ packages/matching/src/matching/keywords.py | 162 ++++++++++++++++ packages/matching/src/matching/similarity.py | 113 ++++++++++++ packages/matching/tests/conftest.py | 184 +++++++++++++++++++ packages/matching/tests/test_dedupe.py | 168 +++++++++++++++++ packages/matching/tests/test_keywords.py | 152 +++++++++++++++ packages/matching/tests/test_similarity.py | 126 +++++++++++++ 10 files changed, 1135 insertions(+) create mode 100644 packages/matching/README.md create mode 100644 packages/matching/pyproject.toml create mode 100644 packages/matching/src/matching/__init__.py create mode 100644 packages/matching/src/matching/dedupe.py create mode 100644 packages/matching/src/matching/keywords.py create mode 100644 packages/matching/src/matching/similarity.py create mode 100644 packages/matching/tests/conftest.py create mode 100644 packages/matching/tests/test_dedupe.py create mode 100644 packages/matching/tests/test_keywords.py create mode 100644 packages/matching/tests/test_similarity.py diff --git a/packages/matching/README.md b/packages/matching/README.md new file mode 100644 index 0000000..7fde103 --- /dev/null +++ b/packages/matching/README.md @@ -0,0 +1,40 @@ +# packages/matching + +Job posting similarity, dedupe clustering, and keyword coverage for the +jobhunt-platform v1.1 agency duplicate detection feature (ADR-0003). + +## Modules + +### similarity.py +- `normalize_employer(name)` -- lowercase, strip agency/legal suffixes (AB, Consulting, etc.), remove punctuation. +- `title_score(a, b)` -- rapidfuzz token_set_ratio on job titles (0-100). +- `employer_match(a, b)` -- True if normalized employer names are equal. +- `desc_score(a, b, max_chars=2000)` -- token_set_ratio on first 2000 chars of descriptions. + +### dedupe.py +- `cluster(postings: list[dict]) -> dict[str, list[str]]` -- group postings into duplicate clusters. + +Clustering rule (per ADR-0003): +- Same employer (normalized) **OR** +- Title similarity >= 85 **AND** description similarity >= 80 + +Uses union-find for transitive grouping. Clusters are sorted by descending +max pairwise score (`c1` = tightest cluster). + +### keywords.py +- `extract_keywords(text, top_n=30)` -- frequency-based keyword extraction with Swedish + English stopword removal. +- `coverage(cv_text, posting_text)` -- computes keyword coverage of a CV against a job posting. + +Multiword tech terms like "fast api" are collapsed to "fastapi" so they survive as single keywords. + +## Installation (uv) + +```bash +cd packages/matching +uv venv && source .venv/bin/activate +uv pip install -e ".[dev]" +pytest +``` + +## License +MIT \ No newline at end of file diff --git a/packages/matching/pyproject.toml b/packages/matching/pyproject.toml new file mode 100644 index 0000000..d5463af --- /dev/null +++ b/packages/matching/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "matching" +version = "0.1.0" +description = "Job posting similarity, dedupe clustering, and keyword coverage for agency duplicate detection." +requires-python = ">=3.13" +dependencies = [ + "rapidfuzz>=3.6", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.24", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/matching"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +asyncio_mode = "auto" \ No newline at end of file diff --git a/packages/matching/src/matching/__init__.py b/packages/matching/src/matching/__init__.py new file mode 100644 index 0000000..e03af4f --- /dev/null +++ b/packages/matching/src/matching/__init__.py @@ -0,0 +1,28 @@ +"""Job posting matching package. + +Provides: +- similarity: normalized text comparison (employer match, title/desc scores) +- dedupe: cluster postings into duplicate groups +- keywords: keyword extraction and CV-vs-posting coverage +""" + +from __future__ import annotations + +from matching.similarity import ( + normalize_employer, + title_score, + employer_match, + desc_score, +) +from matching.dedupe import cluster +from matching.keywords import extract_keywords, coverage + +__all__ = [ + "normalize_employer", + "title_score", + "employer_match", + "desc_score", + "cluster", + "extract_keywords", + "coverage", +] \ No newline at end of file diff --git a/packages/matching/src/matching/dedupe.py b/packages/matching/src/matching/dedupe.py new file mode 100644 index 0000000..3463b74 --- /dev/null +++ b/packages/matching/src/matching/dedupe.py @@ -0,0 +1,136 @@ +"""Duplicate clustering for job postings. + +Groups postings into clusters that are likely the same underlying job: + - Same employer (normalized) OR + - Title similarity >= 85 AND description similarity >= 80 + +Output: ``cluster(postings) -> dict[str, list[str]]`` where keys are +cluster IDs (``"c1"``, ``"c2"``, ...) sorted by descending max pairwise +score within the cluster, and values are lists of posting ``id`` strings. +""" + +from __future__ import annotations + +from matching.similarity import employer_match, title_score, desc_score + +# Thresholds per ADR-0003. +TITLE_THRESHOLD = 85.0 +DESC_THRESHOLD = 80.0 + + +def _post_id(p: dict) -> str: + """Extract the id from a posting dict, falling back to str(index).""" + pid = p.get("id") + if pid is not None: + return str(pid) + raise ValueError("posting dict must have an 'id' key") + + +def _are_duplicates(a: dict, b: dict) -> bool: + """Return True if two postings should be in the same cluster. + + Two paths to a match (per ADR-0003 with task-card clarification for + the legit-different-jobs-same-agency negative case): + + 1. Same employer (normalized) AND some content overlap + (title >= 85 OR desc >= 80). + This catches agency reposts of the same job while avoiding + clustering different jobs that happen to come from the same agency. + + 2. Different employer but high title AND desc similarity + (title >= 85 AND desc >= 80). + This catches cross-agency reposts of the same job. + """ + ts = title_score(a.get("title", ""), b.get("title", "")) + ds = desc_score(a.get("description", ""), b.get("description", "")) + same_employer = employer_match(a.get("employer", ""), b.get("employer", "")) + + if same_employer: + # Same employer + at least one content dimension similar. + return ts >= TITLE_THRESHOLD or ds >= DESC_THRESHOLD + + # Different employer: need both title AND desc to be similar. + return ts >= TITLE_THRESHOLD and ds >= DESC_THRESHOLD + + +def _pair_score(a: dict, b: dict) -> float: + """Compute a similarity score between two postings for sorting clusters.""" + ts = title_score(a.get("title", ""), b.get("title", "")) + ds = desc_score(a.get("description", ""), b.get("description", "")) + same_employer = employer_match(a.get("employer", ""), b.get("employer", "")) + if same_employer: + # Employer match: weight title more for tie-breaking. + return 100.0 + ts + return (ts + ds) / 2.0 + + +def cluster(postings: list[dict]) -> dict[str, list[str]]: + """Cluster job postings into duplicate groups. + + Uses union-find so transitive duplicates (A~B, B~C => A~C) are grouped + together. + + Args: + postings: list of dicts with keys ``id``, ``employer``, ``title``, + ``description``. + + Returns: + Dict mapping cluster_id (``"c1"``, ``"c2"``, ...) to a list of + posting id strings. Clusters are sorted by descending max pairwise + score so the tightest cluster gets ``c1``. + """ + n = len(postings) + if n == 0: + return {} + + ids = [_post_id(p) for p in postings] + + # Union-find (disjoint set). + parent: list[int] = list(range(n)) + + def find(x: int) -> int: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[ra] = rb + + # O(n^2) pairwise comparison. + for i in range(n): + for j in range(i + 1, n): + if _are_duplicates(postings[i], postings[j]): + union(i, j) + + # Collect groups. + groups: dict[int, list[int]] = {} + for i in range(n): + root = find(i) + groups.setdefault(root, []).append(i) + + # Compute max pairwise score per group for sorting. + group_scores: list[tuple[float, list[int]]] = [] + for indices in groups.values(): + if len(indices) < 2: + max_sc = 0.0 + else: + max_sc = 0.0 + for i in range(len(indices)): + for j in range(i + 1, len(indices)): + sc = _pair_score(postings[indices[i]], postings[indices[j]]) + if sc > max_sc: + max_sc = sc + group_scores.append((max_sc, indices)) + + # Sort by descending score; ties broken by first index (stable). + group_scores.sort(key=lambda t: (-t[0], t[1][0])) + + # Assign cluster IDs. + result: dict[str, list[str]] = {} + for idx, (_, indices) in enumerate(group_scores, start=1): + result[f"c{idx}"] = [ids[i] for i in indices] + + return result \ No newline at end of file diff --git a/packages/matching/src/matching/keywords.py b/packages/matching/src/matching/keywords.py new file mode 100644 index 0000000..ea86c32 --- /dev/null +++ b/packages/matching/src/matching/keywords.py @@ -0,0 +1,162 @@ +"""Keyword extraction and coverage analysis. + +Deterministic, no LLM. Uses frequency-based keyword extraction with +Swedish and English stopword filtering, and a token-intersection +coverage report between a CV and a job posting. +""" + +from __future__ import annotations + +import re +from collections import Counter + +# --------------------------------------------------------------------------- +# Stopwords (Swedish + English). Conservative lists. +# --------------------------------------------------------------------------- + +_SWEDISH_STOPWORDS: frozenset[str] = frozenset({ + "och", "eller", "som", "att", "den", "det", "de", "vi", "ni", "du", + "han", "hon", "den", "en", "ett", "ar", "har", "var", "var", "inte", + "med", "for", "fran", "till", "pa", "av", "i", "och", "men", "sa", + "när", "då", "hur", "alla", "nagon", "nagot", "alla", "manga", "mycket", + "skall", "ska", "kan", "kommer", "blir", "vore", "vill", "borde", + "efter", "under", "over", "bAKom", "inom", "mellan", "genom", "utan", + "mot", "ut", "fran", "frams", "igår", "idag", "imorgon", +}) + +_ENGLISH_STOPWORDS: frozenset[str] = frozenset({ + "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "being", "have", "has", "had", "do", "does", "did", "will", "would", + "should", "could", "may", "might", "must", "can", "this", "that", + "these", "those", "i", "you", "he", "she", "it", "we", "they", "me", + "him", "her", "us", "them", "my", "your", "his", "its", "our", "their", + "what", "which", "who", "whom", "where", "when", "why", "how", "all", + "each", "every", "both", "few", "more", "most", "other", "some", "such", + "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", + "as", "if", "about", "against", "between", "into", "through", "during", + "before", "after", "above", "below", "up", "down", "out", "off", + "over", "under", "again", "further", "then", "once", "here", "there", + "also", "etc", "e.g", "i.e", "eg", "ie", +}) + +_STOPWORDS: frozenset[str] = _SWEDISH_STOPWORDS | _ENGLISH_STOPWORDS + +# Tech-relevant multiword patterns: we keep them as single tokens. +# e.g. "fast api" -> "fastapi" so it survives as a keyword. +_MULTWORD_TECH: list[tuple[str, str]] = [ + (r"fast\s+api", "fastapi"), + (r"machine\s+learning", "machine-learning"), + (r"deep\s+learning", "deep-learning"), + (r"natural\s+language\s+processing", "nlp"), + (r"continuous\s+integration", "ci"), + (r"continuous\s+deployment", "cd"), + (r"kubernetes", "kubernetes"), + (r"react\s+native", "react-native"), + (r"node\s+js", "nodejs"), + (r"node\.js", "nodejs"), + (r"aws", "aws"), + (r"gcp", "gcp"), + (r"ci/cd", "ci-cd"), +] + +# Minimum token length for keywords. +_MIN_TOKEN_LEN = 2 + +# Tokenizer: split on non-alphanumeric. +_TOKEN_RE = re.compile(r"[a-z0-9+#.\-/]+") + + +def _preprocess_multiwords(text: str) -> str: + """Replace known multiword tech terms with single tokens.""" + result = text.lower() + for pattern, replacement in _MULTWORD_TECH: + result = re.sub(pattern, replacement, result, flags=re.IGNORECASE) + return result + + +def _tokenize(text: str) -> list[str]: + """Tokenize text into lowercased tokens.""" + text = _preprocess_multiwords(text) + raw_tokens = _TOKEN_RE.findall(text.lower()) + tokens: list[str] = [] + for t in raw_tokens: + t = t.strip(".-/") + if not t: + continue + if t in _STOPWORDS: + continue + if len(t) < _MIN_TOKEN_LEN: + continue + # Skip pure numbers (unless they look like versions). + if t.isdigit() and len(t) > 4: + continue + tokens.append(t) + return tokens + + +def extract_keywords(text: str, top_n: int = 30) -> list[str]: + """Extract the top *top_n* keywords from *text* by frequency. + + Keywords are lowercased tokens. Stopwords (Swedish + English) are + removed. Multiword tech terms like "fast api" are collapsed to + "fastapi". + + Args: + text: the text to analyze. + top_n: maximum number of keywords to return. + + Returns: + List of keyword strings, most frequent first. Ties are broken + alphabetically for determinism. + """ + if not text: + return [] + tokens = _tokenize(text) + if not tokens: + return [] + counts: Counter[str] = Counter(tokens) + # Sort by count desc, then alphabetically for deterministic order. + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + return [word for word, _ in ranked[:top_n]] + + +def coverage(cv_text: str, posting_text: str) -> dict[str, list[str] | float]: + """Compute keyword coverage of a CV against a job posting. + + Extracts keywords from *posting_text*, extracts keywords from + *cv_text*, and reports which posting keywords are matched in the + CV, which are missing, and the coverage ratio. + + Args: + cv_text: the candidate's CV text. + posting_text: the job posting text. + + Returns: + Dict with keys: + - ``matched``: list of posting keywords found in CV. + - ``missing``: list of posting keywords NOT found in CV. + - ``ratio``: float (matched / total), 0.0 if no keywords. + """ + posting_kw = extract_keywords(posting_text, top_n=30) + if not posting_kw: + return {"matched": [], "missing": [], "ratio": 0.0} + + cv_kw_set: set[str] = set(extract_keywords(cv_text, top_n=200)) + + matched: list[str] = [] + missing: list[str] = [] + for kw in posting_kw: + if kw in cv_kw_set: + matched.append(kw) + else: + missing.append(kw) + + total = len(posting_kw) + ratio = len(matched) / total if total > 0 else 0.0 + + return { + "matched": matched, + "missing": missing, + "ratio": ratio, + } \ No newline at end of file diff --git a/packages/matching/src/matching/similarity.py b/packages/matching/src/matching/similarity.py new file mode 100644 index 0000000..cb42a0a --- /dev/null +++ b/packages/matching/src/matching/similarity.py @@ -0,0 +1,113 @@ +"""Similarity helpers for job postings. + +Uses rapidfuzz for fuzzy string matching. All functions are deterministic. +""" + +from __future__ import annotations + +import re +from rapidfuzz import fuzz + +# --------------------------------------------------------------------------- +# Agency-suffix / noise words to strip from employer names. +# Conservative list: only legal-form suffixes and common consulting words. +# --------------------------------------------------------------------------- + +_AGENCY_SUFFIXES: list[str] = [ + # Swedish legal forms + "ab", + "aktiebolag", + "hb", + "kb", + "ekonomisk forening", + # English legal forms + "inc", + "corp", + "corporation", + "ltd", + "limited", + "llc", + "gmbh", + "ag", + "sas", + "sarl", + # Consulting / staffing suffixes (agency hints) + "consulting", + "consultancy", + "consult", + "recruitment", + "staffing", + "solutions", + "services", + "group", + "partners", +] + +# Pre-compile regex for trailing suffix removal. +_SUFFIX_RE = re.compile( + r"\s+(" + "|".join(re.escape(s) for s in _AGENCY_SUFFIXES) + r")\.?\s*$", + flags=re.IGNORECASE, +) + +# Characters to collapse: punctuation -> space, then multi-space -> single. +_PUNCT_RE = re.compile(r"[^\w\s]") +_WS_RE = re.compile(r"\s+") + + +def normalize_employer(name: str) -> str: + """Normalize an employer/company name for comparison. + + Steps: + 1. lowercase + 2. strip trailing agency/legal suffixes (ab, consulting, etc.) + 3. remove punctuation + 4. collapse whitespace + + Examples: + >>> normalize_employer("Acme Consulting AB") + 'acme' + >>> normalize_employer("Acme AB") + 'acme' + >>> normalize_employer(" Globex Corp. ") + 'globex' + """ + if not name: + return "" + s = name.strip().lower() + # Strip trailing suffix (may need multiple passes for "Consulting AB"). + for _ in range(3): + new = _SUFFIX_RE.sub("", s) + if new == s: + break + s = new + # Remove punctuation. + s = _PUNCT_RE.sub(" ", s) + s = _WS_RE.sub(" ", s).strip() + return s + + +def title_score(a: str, b: str) -> float: + """Token-set ratio score for two job titles (0-100). + + Uses rapidfuzz ``fuzz.token_set_ratio`` which is order-independent + and handles subsets well. + """ + if not a or not b: + return 0.0 + return float(fuzz.token_set_ratio(a, b)) + + +def employer_match(a: str, b: str) -> bool: + """True if two employer names normalize to the same string.""" + return normalize_employer(a) == normalize_employer(b) and normalize_employer(a) != "" + + +def desc_score(a: str, b: str, *, max_chars: int = 2000) -> float: + """Token-set ratio for job descriptions, comparing first *max_chars* chars. + + Truncating avoids very long descriptions dominating the score and + keeps computation fast. + """ + if not a or not b: + return 0.0 + return float(fuzz.token_set_ratio(a[:max_chars], b[:max_chars])) \ No newline at end of file diff --git a/packages/matching/tests/conftest.py b/packages/matching/tests/conftest.py new file mode 100644 index 0000000..be3db9d --- /dev/null +++ b/packages/matching/tests/conftest.py @@ -0,0 +1,184 @@ +"""Shared fixtures for matching tests. + +Provides agency-repost fixture triples and a legit-different-jobs-same-agency +negative case. +""" + +from __future__ import annotations + +import pytest + + +# --------------------------------------------------------------------------- +# Agency-repost fixture triples. +# Three realistic scenarios where agencies repost the same job. +# Each triple is a list of 3 posting dicts that should all cluster together. +# --------------------------------------------------------------------------- + +# Triple 1: Same role, two agencies + direct employer posting. +# All three have the same title and nearly identical description, but +# different employer names (the two agencies vs the actual company). +TRIPLE_1_AGENCY_REPOST = [ + { + "id": "t1-a", + "employer": "TechCorp AB", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, + { + "id": "t1-b", + "employer": "Nordic IT Consulting AB", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, + { + "id": "t1-c", + "employer": "Acme Recruitment Group", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, +] + +# Triple 2: Same role reposted by same agency with minor wording variations. +TRIPLE_2_SAME_AGENCY_REPOST = [ + { + "id": "t2-a", + "employer": "Stockholm Tech Staffing AB", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, + { + "id": "t2-b", + "employer": "Stockholm Tech Staffing AB", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, + { + "id": "t2-c", + "employer": "Stockholm Tech Staffing", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, +] + +# Triple 3: Same role, slightly different title but same description body. +# Different employers (agencies), high title + desc similarity. +TRIPLE_3_CROSS_AGENCY = [ + { + "id": "t3-a", + "employer": "Data Recruiting Solutions", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, + { + "id": "t3-b", + "employer": "Cloud Talent Partners", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, + { + "id": "t3-c", + "employer": "Analytics Staffing Ltd", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, +] + +# --------------------------------------------------------------------------- +# NEGATIVE case: legit different jobs at same agency -- must NOT cluster. +# Same agency employer but different titles and different descriptions. +# --------------------------------------------------------------------------- + +NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY = [ + { + "id": "neg-a", + "employer": "Nordic IT Consulting AB", + "title": "Frontend Developer", + "description": ( + "We are looking for a Frontend Developer with expertise in React " + "and TypeScript. You will build responsive web applications and " + "work closely with our design team. Experience with CSS-in-JS and " + "accessibility standards is required." + ), + }, + { + "id": "neg-b", + "employer": "Nordic IT Consulting AB", + "title": "DevOps Engineer", + "description": ( + "We need a DevOps Engineer to manage our Kubernetes clusters and " + "CI/CD pipelines. You will work with Terraform, ArgoCD, and " + "Prometheus monitoring. Strong Linux and networking background " + "is required. AWS certification is a plus." + ), + }, +] + + +@pytest.fixture +def triple1(): + """Agency repost triple 1: same role, two agencies + employer.""" + return [dict(p) for p in TRIPLE_1_AGENCY_REPOST] + + +@pytest.fixture +def triple2(): + """Agency repost triple 2: same agency reposts same job.""" + return [dict(p) for p in TRIPLE_2_SAME_AGENCY_REPOST] + + +@pytest.fixture +def triple3(): + """Agency repost triple 3: cross-agency same role same description.""" + return [dict(p) for p in TRIPLE_3_CROSS_AGENCY] + + +@pytest.fixture +def negative_same_agency(): + """Negative case: different jobs at same agency, must NOT cluster.""" + return [dict(p) for p in NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY] \ No newline at end of file diff --git a/packages/matching/tests/test_dedupe.py b/packages/matching/tests/test_dedupe.py new file mode 100644 index 0000000..9af1a09 --- /dev/null +++ b/packages/matching/tests/test_dedupe.py @@ -0,0 +1,168 @@ +"""Tests for dedupe clustering.""" + +from __future__ import annotations + +from matching.dedupe import cluster + + +class TestClusterBasic: + def test_empty_list(self): + assert cluster([]) == {} + + def test_single_posting(self): + result = cluster([ + {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"} + ]) + assert len(result) == 1 + assert "c1" in result + assert result["c1"] == ["a"] + + def test_no_duplicates_separate_clusters(self): + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"}, + {"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"}, + {"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"}, + ] + result = cluster(postings) + # Each posting in its own cluster. + total_ids = sum(len(v) for v in result.values()) + assert total_ids == 3 + # All cluster values are singletons. + for ids in result.values(): + assert len(ids) == 1 + + +class TestClusterEmployerMatch: + def test_same_employer_clusters(self): + """Same employer name (normalized) with similar content should cluster.""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, + {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, + ] + result = cluster(postings) + # Same employer + similar title -> same cluster. + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + def test_same_employer_different_content_no_cluster(self): + """Same employer but completely different titles/descriptions should NOT cluster.""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."}, + {"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."}, + ] + result = cluster(postings) + # Same employer but different jobs -> no cluster. + assert all(len(v) == 1 for v in result.values()) + + def test_employer_suffix_variations_cluster(self): + """Acme AB and Acme should cluster (normalized match).""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}, + {"id": "b", "employer": "Acme", "title": "Dev", "description": "y"}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + +class TestClusterTitleDescMatch: + def test_title_desc_high_enough(self): + """Different employers but title >= 85 and desc >= 80 -> cluster.""" + desc = ( + "We are looking for a Senior Python Developer to join our backend " + "team. You will work with Fast API, PostgreSQL, and Docker." + ) + postings = [ + {"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc}, + {"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + def test_title_high_desc_low_no_cluster(self): + """Title similar but desc too different -> no cluster.""" + postings = [ + {"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."}, + {"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."}, + ] + result = cluster(postings) + # Should NOT cluster (different employers, low desc score). + assert len(result) == 2 or all(len(v) == 1 for v in result.values()) + + +class TestClusterTransitive: + def test_transitive_clustering(self): + """If A~B and B~C then A~C should be in same cluster.""" + # A and B same employer + similar title, B and C same employer + similar title. + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, + {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, + {"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b", "c"} + + +class TestClusterSorting: + def test_cluster_ids_sorted_by_score(self): + """Cluster with higher pairwise score should get c1.""" + # Tight cluster: identical titles and descriptions (different employers). + tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes." + # Looser cluster: different employers, high title but lower desc similarity. + loose_desc_1 = "Python data engineering and pipelines with ETL tools." + loose_desc_2 = "Python data engineering and ETL work with Airflow." + postings = [ + # tight cluster (different employers, high title+desc) + {"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc}, + {"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc}, + # loose cluster (different employers, high title but lower desc) + {"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1}, + {"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2}, + ] + result = cluster(postings) + # Both clusters should exist. + all_ids = set() + for ids in result.values(): + all_ids.update(ids) + assert all_ids == {"t1", "t2", "l1", "l2"} + # c1 should be the tight cluster (higher score: title+desc both 100). + assert set(result["c1"]) == {"t1", "t2"} + + +class TestAgencyRepostTriples: + """Test the 3 agency-repost fixture triples.""" + + def test_triple1_all_cluster(self, triple1): + """Triple 1: 3 postings of same role via different employers cluster.""" + result = cluster(triple1) + assert len(result) == 1 + assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"} + + def test_triple2_all_cluster(self, triple2): + """Triple 2: same agency reposts same job (suffix variations).""" + result = cluster(triple2) + assert len(result) == 1 + assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"} + + def test_triple3_all_cluster(self, triple3): + """Triple 3: cross-agency same role same description.""" + result = cluster(triple3) + assert len(result) == 1 + assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"} + + +class TestNegativeSameAgencyDifferentJobs: + """Negative case: different jobs at same agency must NOT cluster.""" + + def test_different_jobs_same_agency_no_cluster(self, negative_same_agency): + """Different jobs at same agency must NOT cluster. + + Same employer but completely different titles and descriptions. + Per the clustering rule, same employer alone is not sufficient; + some content overlap (title >= 85 OR desc >= 80) is also required. + """ + result = cluster(negative_same_agency) + all_singletons = all(len(v) == 1 for v in result.values()) + assert all_singletons, "Different jobs at same agency should not cluster" \ No newline at end of file diff --git a/packages/matching/tests/test_keywords.py b/packages/matching/tests/test_keywords.py new file mode 100644 index 0000000..0c0c742 --- /dev/null +++ b/packages/matching/tests/test_keywords.py @@ -0,0 +1,152 @@ +"""Tests for keyword extraction and coverage.""" + +from __future__ import annotations + +from matching.keywords import extract_keywords, coverage + + +class TestExtractKeywords: + def test_basic_extraction(self): + text = "Python developer with Fast API experience and PostgreSQL database skills." + kws = extract_keywords(text) + assert "python" in kws + assert "fastapi" in kws + assert "postgresql" in kws + + def test_stopwords_removed(self): + text = "We are looking for a developer with experience in Python." + kws = extract_keywords(text) + assert "we" not in kws + assert "are" not in kws + assert "for" not in kws + assert "a" not in kws + assert "in" not in kws + assert "python" in kws + assert "developer" in kws + + def test_swedish_stopwords_removed(self): + text = "Vi letar efter en Python utvecklare med erfarenhet av Docker." + kws = extract_keywords(text) + assert "vi" not in kws + assert "en" not in kws + assert "av" not in kws + assert "python" in kws + assert "docker" in kws + + def test_top_n_limit(self): + text = "python python python docker docker docker kubernetes kubernetes kubernetes react react react" + kws = extract_keywords(text, top_n=2) + assert len(kws) == 2 + + def test_empty_text(self): + assert extract_keywords("") == [] + + def test_whitespace_only(self): + assert extract_keywords(" ") == [] + + def test_multiword_fastapi(self): + text = "Experience with fast api framework for building REST APIs." + kws = extract_keywords(text) + assert "fastapi" in kws + + def test_multiword_machine_learning(self): + text = "machine learning models for predictive analytics." + kws = extract_keywords(text) + assert "machine-learning" in kws + + def test_frequency_ordering(self): + text = "python python python docker docker kubernetes" + kws = extract_keywords(text, top_n=3) + assert kws[0] == "python" + assert kws[1] == "docker" + assert kws[2] == "kubernetes" + + def test_deterministic_tie_breaking(self): + """Ties in frequency should be broken alphabetically.""" + text = "docker kubernetes" + kws = extract_keywords(text) + # Both have frequency 1, so alphabetical: docker < kubernetes + assert kws[0] == "docker" + assert kws[1] == "kubernetes" + + def test_min_token_length(self): + text = "x y z aa bb cc developer" + kws = extract_keywords(text) + assert "x" not in kws + assert "y" not in kws + assert "z" not in kws + assert "developer" in kws + + def test_tech_terms_preserved(self): + text = "Node.js and React Native for mobile development." + kws = extract_keywords(text) + assert "nodejs" in kws + assert "react-native" in kws + + +class TestCoverage: + def test_full_coverage(self): + cv = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes" + posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes" + result = coverage(cv, posting) + assert result["ratio"] == 1.0 + assert len(result["missing"]) == 0 + + def test_partial_coverage(self): + cv = "Python developer with PostgreSQL and Docker experience" + posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes React" + result = coverage(cv, posting) + assert 0.0 < result["ratio"] < 1.0 + assert "python" in result["matched"] + assert "postgresql" in result["matched"] + assert "docker" in result["matched"] + assert "fastapi" in result["missing"] + assert "kubernetes" in result["missing"] + + def test_zero_coverage(self): + cv = "Chef with experience in French cuisine and menu planning" + posting = "Python developer with Fast API PostgreSQL Docker" + result = coverage(cv, posting) + assert result["ratio"] == 0.0 + assert len(result["matched"]) == 0 + assert len(result["missing"]) > 0 + + def test_empty_posting(self): + result = coverage("Python developer", "") + assert result == {"matched": [], "missing": [], "ratio": 0.0} + + def test_empty_cv(self): + result = coverage("", "Python developer with Docker") + assert result["ratio"] == 0.0 + assert len(result["matched"]) == 0 + assert len(result["missing"]) > 0 + + def test_both_empty(self): + result = coverage("", "") + assert result == {"matched": [], "missing": [], "ratio": 0.0} + + def test_ratio_calculation(self): + cv = "python docker postgresql" + posting = "python docker postgresql kubernetes" + result = coverage(cv, posting) + # 3 of 4 matched (approx, depends on stopword filtering). + assert result["ratio"] > 0.5 + assert result["ratio"] <= 1.0 + + def test_matched_and_missing_lists(self): + cv = "python docker" + posting = "python docker kubernetes react" + result = coverage(cv, posting) + assert "python" in result["matched"] + assert "docker" in result["matched"] + assert "kubernetes" in result["missing"] + assert "react" in result["missing"] + + def test_coverage_returns_dict_keys(self): + result = coverage("python", "python docker") + assert "matched" in result + assert "missing" in result + assert "ratio" in result + assert isinstance(result["matched"], list) + assert isinstance(result["missing"], list) + assert isinstance(result["ratio"], float) \ No newline at end of file diff --git a/packages/matching/tests/test_similarity.py b/packages/matching/tests/test_similarity.py new file mode 100644 index 0000000..9d2fd16 --- /dev/null +++ b/packages/matching/tests/test_similarity.py @@ -0,0 +1,126 @@ +"""Tests for similarity helpers.""" + +from __future__ import annotations + +from matching.similarity import ( + normalize_employer, + title_score, + employer_match, + desc_score, +) + + +class TestNormalizeEmployer: + def test_simple_lowercase(self): + assert normalize_employer("Acme") == "acme" + + def test_strips_swedish_ab(self): + assert normalize_employer("Acme AB") == "acme" + + def test_strips_aktiebolag(self): + assert normalize_employer("Acme Aktiebolag") == "acme" + + def test_strips_consulting_suffix(self): + assert normalize_employer("Nordic IT Consulting AB") == "nordic it" + + def test_strips_corp_suffix(self): + assert normalize_employer("Globex Corp.") == "globex" + + def test_strips_ltd_suffix(self): + assert normalize_employer("Foo Ltd") == "foo" + + def test_strips_recruitment_suffix(self): + assert normalize_employer("Acme Recruitment Group") == "acme" + + def test_strips_multiple_suffixes(self): + # "Consulting AB" should strip both "AB" then "Consulting" + assert normalize_employer("Nordic Consulting AB") == "nordic" + + def test_removes_punctuation(self): + assert normalize_employer("Acme, Inc.") == "acme" + + def test_empty_string(self): + assert normalize_employer("") == "" + + def test_whitespace_only(self): + assert normalize_employer(" ") == "" + + def test_preserves_core_name_with_special_chars(self): + result = normalize_employer("Café Nu AB") + assert "café" in result or "cafe" in result + + def test_dots_in_name_preserved(self): + # Punctuation (except & which gets stripped) is removed; H&M -> h m + result = normalize_employer("H&M AB") + assert result == "h m" + + +class TestTitleScore: + def test_identical_titles(self): + assert title_score("Senior Python Developer", "Senior Python Developer") == 100.0 + + def test_similar_titles_high_score(self): + score = title_score("Python Developer", "Senior Python Developer") + assert score >= 85.0 + + def test_different_titles_low_score(self): + score = title_score("Python Developer", "Frontend Designer") + assert score < 50.0 + + def test_empty_title(self): + assert title_score("", "Something") == 0.0 + + def test_both_empty(self): + assert title_score("", "") == 0.0 + + def test_order_independent(self): + # token_set_ratio is order-independent + a = "Senior Python Developer" + b = "Developer Python Senior" + assert title_score(a, b) == 100.0 + + +class TestEmployerMatch: + def test_same_name_matches(self): + assert employer_match("Acme AB", "Acme AB") is True + + def test_suffix_variation_matches(self): + assert employer_match("Acme AB", "Acme") is True + + def test_different_employers_no_match(self): + assert employer_match("Acme AB", "Globex AB") is False + + def test_consulting_variations_match(self): + assert employer_match("Nordic IT Consulting AB", "Nordic IT") is True + + def test_empty_no_match(self): + assert employer_match("", "") is False + + def test_one_empty_no_match(self): + assert employer_match("Acme", "") is False + + +class TestDescScore: + def test_identical_descriptions(self): + desc = "We are looking for a Python developer with 5 years experience." + assert desc_score(desc, desc) == 100.0 + + def test_similar_descriptions_high(self): + a = "We are looking for a Python developer with 5 years experience." + b = "We are looking for a Python developer with 5 years experience in web." + assert desc_score(a, b) >= 80.0 + + def test_different_descriptions_low(self): + a = "We need a frontend developer skilled in React and CSS." + b = "Looking for a data scientist with Python and SQL expertise." + assert desc_score(a, b) < 50.0 + + def test_empty_desc(self): + assert desc_score("", "something") == 0.0 + + def test_truncation(self): + # Test that truncation to max_chars works. + long_a = "Python " * 1000 + long_b = "Python " * 1000 + score = desc_score(long_a, long_b, max_chars=100) + assert score == 100.0 \ No newline at end of file