Compare commits

..

No commits in common. "7bd9e5bfc6b7e4cef81a491dea09eaca4987af7a" and "5e539ba713590ac10192d008df779f1221eb8809" have entirely different histories.

24 changed files with 6 additions and 2893 deletions

View file

@ -73,7 +73,6 @@ def reset_database(database_url: str | None = None) -> None:
conn.execute(
"""
DROP TABLE IF EXISTS task_run, outbox, approval, artifact,
email_suggestion, notification_log,
application, job_posting, cv_section, profile, schema_migrations
CASCADE
"""

View file

@ -1,468 +0,0 @@
"""IMAP email watch: polls UNSEEN messages and classifies them.
Uses stdlib imaplib (SSL). Enabled only when EMAIL_WATCH_ENABLED=true.
Config: IMAP_HOST, IMAP_PORT, IMAP_USER, IMAP_PASS.
Flow:
1. Connect via IMAP SSL.
2. Fetch UNSEEN messages since last poll.
3. For each message, match sender domain + subject/body keywords to
open applications (status in sent/interviewing).
4. Classify via LLM task 'email_classify' -> {classification, state_proposal, reason}.
5. Insert email_suggestion rows (skip noise and dedupe by from+subject+day).
"""
from __future__ import annotations
import email
import email.utils
import hashlib
import imaplib
import logging
import os
from datetime import datetime, timezone
from typing import Any, Sequence
from app.db import execute, fetch_all, fetch_one
from app import llm
logger = logging.getLogger(__name__)
VALID_CLASSIFICATIONS = frozenset({
"interview_invite",
"rejection",
"question",
"noise",
})
# Keywords for cheap pre-matching before LLM classify
INTERVIEW_KEYWORDS = ("interview", "invite", "meeting", "schedule", "call")
REJECTION_KEYWORDS = ("regret", "unfortunately", "not moving", "rejection", "position has been filled")
QUESTION_KEYWORDS = ("question", "clarif", "additional", "could you", "please provide")
NOISE_KEYWORDS = ("newsletter", "unsubscribe", "promotion", "advert", "offer")
def is_email_watch_enabled() -> bool:
"""Check if email watch is enabled."""
return os.environ.get("EMAIL_WATCH_ENABLED", "false").lower() in (
"true",
"1",
"yes",
)
def _extract_sender_domain(from_addr: str) -> str:
"""Extract the domain from an email From header."""
parsed = email.utils.parseaddr(from_addr)
addr = parsed[1] or from_addr
parts = addr.split("@")
if len(parts) >= 2:
return parts[-1].lower().strip()
return ""
def _build_snippet(body: str, max_len: int = 300) -> str:
"""Truncate body to a snippet."""
body = body.replace("\r", " ").replace("\n", " ").strip()
if len(body) > max_len:
return body[:max_len] + "..."
return body
def _parse_email_message(raw_bytes: bytes) -> dict[str, str]:
"""Parse raw email bytes into a dict with from, subject, body."""
msg = email.message_from_bytes(raw_bytes)
from_addr = msg.get("From", "")
subject = msg.get("Subject", "")
date_str = msg.get("Date", "")
# Extract body (prefer plain text)
body = ""
if msg.is_multipart():
for part in msg.walk():
ct = part.get_content_type()
if ct == "text/plain":
payload = part.get_payload(decode=True)
if payload:
body = payload.decode("utf-8", errors="replace")
break
if not body:
for part in msg.walk():
if part.get_content_type().startswith("text/"):
payload = part.get_payload(decode=True)
if payload:
body = payload.decode("utf-8", errors="replace")
break
else:
payload = msg.get_payload(decode=True)
if payload:
body = payload.decode("utf-8", errors="replace")
return {
"from": from_addr,
"subject": subject,
"body": body,
"date": date_str,
}
def _parse_date(date_str: str) -> datetime:
"""Parse RFC 2822 date string to UTC datetime. Falls back to now()."""
if date_str:
try:
parsed = email.utils.parsedate_to_datetime(date_str)
if parsed is not None:
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
except (ValueError, TypeError):
pass
return datetime.now(timezone.utc)
def match_application(
from_addr: str,
subject: str,
body: str,
applications: Sequence[dict[str, Any]],
) -> dict[str, Any] | None:
"""Match an email to an open application.
Match by:
1. Company name in subject/body
2. Sender domain in posting URL
Only matches applications with state in ('sent', 'interviewing').
"""
sender_domain = _extract_sender_domain(from_addr)
text_lower = (subject + " " + body).lower()
for app_row in applications:
state = app_row.get("state", "")
if state not in ("sent", "interviewing"):
continue
company = (app_row.get("company") or "").lower()
title = (app_row.get("title") or "").lower()
posting_url = app_row.get("url") or ""
# Match 1: company name in subject/body
if company and len(company) > 2 and company in text_lower:
return app_row
# Match 2: sender domain in posting URL
if sender_domain and sender_domain in (posting_url or "").lower():
return app_row
# Match 3: title keywords in subject (looser)
if title and len(title) > 3:
title_words = [w for w in title.split() if len(w) > 3]
matches = sum(1 for w in title_words if w in text_lower)
if matches >= 2 and len(title_words) >= 2:
return app_row
return None
def classify_email(subject: str, body: str) -> dict[str, Any]:
"""Classify an email via LLM task 'email_classify'.
Returns {classification, state_proposal, reason}.
Falls back to inline keyword heuristics if LLM fails.
"""
text_lower = (subject + " " + body).lower()
result = llm.run_task(
"email_classify",
f"Subject: {subject}\nBody: {body[:1000]}",
)
classification = result.get("classification", "noise")
state_proposal = result.get("state_proposal")
reason = result.get("reason", "")
# Validate
if classification not in VALID_CLASSIFICATIONS:
classification = "noise"
return {
"classification": classification,
"state_proposal": state_proposal,
"reason": reason,
}
def is_duplicate(from_addr: str, subject: str, received_at: datetime) -> bool:
"""Check if a similar email_suggestion already exists (same from+subject+day)."""
day_start = received_at.replace(hour=0, minute=0, second=0, microsecond=0)
row = fetch_one(
"""
SELECT id FROM email_suggestion
WHERE mailbox_from = %s AND subject = %s
AND received_at >= %s AND received_at < %s + interval '1 day'
LIMIT 1
""",
(from_addr, subject, day_start, day_start),
)
return row is not None
def create_email_suggestion(
application_id: str | None,
mailbox_from: str,
subject: str,
snippet: str,
classification: str,
state_proposal: str | None,
received_at: datetime,
) -> dict[str, Any]:
"""Insert an email_suggestion row."""
row = execute(
"""
INSERT INTO email_suggestion
(application_id, mailbox_from, subject, snippet, classification, state_proposal, status, received_at)
VALUES
(%s, %s, %s, %s, %s, %s, 'pending', %s)
RETURNING *
""",
(application_id, mailbox_from, subject, snippet, classification, state_proposal, received_at),
)
if row is None:
raise RuntimeError("insert email_suggestion failed")
return _normalize_suggestion(row)
def _normalize_suggestion(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": str(row["id"]),
"application_id": str(row["application_id"]) if row.get("application_id") else None,
"mailbox_from": row["mailbox_from"],
"subject": row["subject"],
"snippet": row["snippet"],
"classification": row["classification"],
"state_proposal": row.get("state_proposal"),
"status": row["status"],
"received_at": row["received_at"].isoformat() if row.get("received_at") else None,
"created_at": row["created_at"].isoformat() if row.get("created_at") else None,
}
def list_pending_suggestions() -> list[dict[str, Any]]:
"""Return all pending email_suggestions, newest first."""
rows = fetch_all(
"""
SELECT * FROM email_suggestion
WHERE status = 'pending'
ORDER BY received_at DESC
"""
)
return [_normalize_suggestion(r) for r in rows]
def get_suggestion(suggestion_id: str) -> dict[str, Any] | None:
row = fetch_one("SELECT * FROM email_suggestion WHERE id = %s", (suggestion_id,))
if row is None:
return None
return _normalize_suggestion(row)
def update_suggestion_status(
suggestion_id: str,
status: str,
) -> dict[str, Any] | None:
row = execute(
"UPDATE email_suggestion SET status = %s WHERE id = %s RETURNING *",
(status, suggestion_id),
)
if row is None:
return None
return _normalize_suggestion(row)
def get_open_applications() -> list[dict[str, Any]]:
"""Return applications in sent/interviewing state for matching."""
rows = fetch_all(
"""
SELECT a.*, j.company, j.title, j.url
FROM application a
JOIN job_posting j ON a.job_posting_id = j.id
WHERE a.state IN ('sent', 'interviewing')
"""
)
result: list[dict[str, Any]] = []
for row in rows:
result.append({
"id": str(row["id"]),
"state": row["state"],
"company": row.get("company", ""),
"title": row.get("title", ""),
"url": row.get("url", ""),
})
return result
# --- IMAP poll ---
class FakeImap:
"""Test double for imaplib IMAP4_SSL. No network.
Usage:
fake = FakeImap(messages=[(uid1, raw1), (uid2, raw2)])
poll_inbox(fake) # uses fake instead of real connection
"""
def __init__(self, messages: list[tuple[bytes, bytes]] | None = None) -> None:
# messages: list of (uid, raw_email_bytes)
self._messages = messages or []
self._seen_uids: set[bytes] = set()
self.selected = False
def select(self, mailbox: str = "INBOX") -> tuple[str, list[bytes]]:
self.selected = True
count = len(self._messages)
return ("OK", [str(count).encode()])
def search(self, charset: str | None, *criteria: str) -> tuple[str, list[bytes]]:
# Return uids of messages matching criteria (we keep it simple)
uids = [uid for uid, _ in self._messages]
return ("OK", [b" ".join(uids)])
def fetch(self, uid: bytes, parts: str) -> tuple[str, list[tuple[bytes, bytes]]]:
for msg_uid, raw in self._messages:
if msg_uid == uid:
return ("OK", [(uid, raw)])
return ("OK", [])
def store(self, uid: bytes, flags: str, flag_set: str) -> tuple[str, list[bytes]]:
self._seen_uids.add(uid)
return ("OK", [uid])
def close(self) -> tuple[str, list[bytes]]:
self.selected = False
return ("OK", [b""])
def logout(self) -> tuple[str, list[bytes]]:
return ("OK", [b"BYE"])
def _connect_imap() -> Any:
"""Connect to IMAP server using env config."""
host = os.environ.get("IMAP_HOST", "")
port = int(os.environ.get("IMAP_PORT", "993"))
user = os.environ.get("IMAP_USER", "")
password = os.environ.get("IMAP_PASS", "")
conn = imaplib.IMAP4_SSL(host, port)
conn.login(user, password)
return conn
def poll_inbox(conn: Any = None) -> list[dict[str, Any]]:
"""Poll the inbox for UNSEEN messages, classify, and create suggestions.
If conn is provided (e.g. FakeImap for tests), uses it instead of connecting.
Returns list of created suggestions.
No network is used when conn is a FakeImap.
"""
created: list[dict[str, Any]] = []
own_conn = False
if conn is None:
conn = _connect_imap()
own_conn = True
try:
conn.select("INBOX")
status, data = conn.search(None, "UNSEEN")
if status != "OK":
logger.warning("imap_watch: search failed: %s", status)
return created
uids = []
if data and data[0]:
uids = data[0].split()
if not uids:
return created
# Get open applications for matching
applications = get_open_applications()
for uid in uids:
status, fetch_data = conn.fetch(uid, "(RFC822)")
if status != "OK" or not fetch_data:
continue
raw_bytes = b""
for item in fetch_data:
if isinstance(item, tuple) and len(item) >= 2:
raw_bytes = item[1]
break
if not raw_bytes:
continue
parsed = _parse_email_message(raw_bytes)
from_addr = parsed["from"]
subject = parsed["subject"]
body = parsed["body"]
received_at = _parse_date(parsed["date"])
snippet = _build_snippet(body)
# Dedupe: same from+subject+day
if is_duplicate(from_addr, subject, received_at):
logger.debug("imap_watch: dedupe skip: %s / %s", from_addr, subject)
continue
# Match to application
app_row = match_application(from_addr, subject, body, applications)
# Classify
classification_result = classify_email(subject, body)
classification = classification_result["classification"]
state_proposal = classification_result["state_proposal"]
# Skip pure noise (don't create suggestion rows)
if classification == "noise":
continue
suggestion = create_email_suggestion(
application_id=app_row["id"] if app_row else None,
mailbox_from=from_addr,
subject=subject,
snippet=snippet,
classification=classification,
state_proposal=state_proposal,
received_at=received_at,
)
created.append(suggestion)
# Send notification for interview invites
if classification == "interview_invite" and app_row:
from app.notify import send_notification
send_notification(
"email_suggestion",
f"Interview invite from {app_row.get('company', 'unknown')}",
{
"suggestion_id": suggestion["id"],
"application_id": app_row["id"],
"classification": classification,
},
)
# Mark as seen
try:
conn.store(uid, "+FLAGS", "\\Seen")
except Exception:
logger.debug("imap_watch: store failed for uid %s", uid)
finally:
if own_conn:
try:
conn.close()
conn.logout()
except Exception:
pass
return created

View file

@ -123,11 +123,6 @@ MOCK_OUTPUTS: dict[str, dict[str, Any]] = {
"posting requirements. Be specific."
),
},
"email_classify": {
"classification": "interview_invite",
"state_proposal": "interviewing",
"reason": "The email mentions an interview invitation.",
},
}

View file

@ -37,13 +37,11 @@ from app.schemas import (
CvSectionOut,
CvSectionUpdate,
DigestItem,
EmailSuggestionOut,
ErrorOut,
InterviewPrepResponse,
JobPostingCreate,
JobPostingOut,
NudgeItem,
NotificationLogOut,
OutboxOut,
OutboxSendRequest,
PostingsFetchRequest,
@ -934,109 +932,4 @@ def seed_demo() -> Any:
"postings": postings_count,
"applications": apps_count,
"sections": sections_count,
}
# --- v1.1: Email Suggestions ---
@app.get("/api/suggestions", response_model=list[EmailSuggestionOut])
def get_suggestions() -> Any:
"""Return all pending email suggestions, newest first."""
from app.imap_watch import list_pending_suggestions
return list_pending_suggestions()
@app.post("/api/suggestions/{suggestion_id}/accept")
def accept_suggestion(suggestion_id: str) -> Any:
"""Accept a suggestion: apply state_proposal via guarded transition.
If the suggestion has an application_id and a state_proposal, apply the
state transition through the normal guard path. Updates last_activity_at.
Marks the suggestion as 'accepted'.
"""
from app.imap_watch import get_suggestion, update_suggestion_status
suggestion = get_suggestion(suggestion_id)
if suggestion is None:
raise HTTPException(status_code=404, detail="Suggestion not found")
if suggestion["status"] != "pending":
raise HTTPException(
status_code=409,
detail={"code": "not_pending", "message": "Suggestion is not pending"},
)
app_id = suggestion.get("application_id")
state_proposal = suggestion.get("state_proposal")
if app_id and state_proposal:
# Apply guarded transition
app_row = repo_app.get_application(app_id)
if app_row is None:
raise HTTPException(status_code=404, detail="Linked application not found")
from_state = app_row["state"]
to_state = state_proposal
from app.statemachine import TransitionContext, check_transition, InvalidTransition
has_score = app_row.get("score") is not None
has_confirmed_approval = False
artifact_hash_match = False
ctx = TransitionContext(
application_id=app_id,
from_state=from_state,
to_state=to_state,
has_score=has_score,
has_confirmed_approval=has_confirmed_approval,
artifact_hash_match=artifact_hash_match,
)
try:
check_transition(ctx)
except InvalidTransition as exc:
raise HTTPException(
status_code=409,
detail={"code": "invalid_transition", "message": str(exc)},
)
updated = repo_app.update_application_state(app_id, to_state)
if updated is None:
raise HTTPException(status_code=500, detail="State update failed")
# Mark suggestion as accepted
result = update_suggestion_status(suggestion_id, "accepted")
if result is None:
raise HTTPException(status_code=500, detail="Failed to update suggestion")
return result
@app.post("/api/suggestions/{suggestion_id}/dismiss")
def dismiss_suggestion(suggestion_id: str) -> Any:
"""Dismiss a suggestion (mark as dismissed)."""
from app.imap_watch import get_suggestion, update_suggestion_status
suggestion = get_suggestion(suggestion_id)
if suggestion is None:
raise HTTPException(status_code=404, detail="Suggestion not found")
if suggestion["status"] != "pending":
raise HTTPException(
status_code=409,
detail={"code": "not_pending", "message": "Suggestion is not pending"},
)
result = update_suggestion_status(suggestion_id, "dismissed")
if result is None:
raise HTTPException(status_code=500, detail="Failed to update suggestion")
return result
# --- v1.1: Notification Log ---
@app.get("/api/notifications/log", response_model=list[NotificationLogOut])
def get_notification_log() -> Any:
"""Return last 50 notification log entries."""
from app.notify import list_notification_log
return list_notification_log(limit=50)
}

View file

@ -1,183 +0,0 @@
"""Notification channels: LogChannel + WebhookChannel.
Protocol-based: NotificationChannel defines send(kind, text, data) -> bool.
LogChannel writes to notification_log (delivered=true always).
WebhookChannel POSTs to NOTIFY_WEBHOOK_URL (2xx=delivered, else error row).
send_notification(kind, text, data) is the public entry point used by
the scheduler and email watch modules.
"""
from __future__ import annotations
import json
import logging
import os
from typing import Any, Protocol
from app.db import execute, fetch_all
logger = logging.getLogger(__name__)
class NotificationChannel(Protocol):
"""Protocol for notification delivery channels."""
def send(self, kind: str, text: str, data: dict[str, Any]) -> bool:
"""Send a notification. Returns True if delivered, False otherwise."""
...
def _normalize_log_row(row: dict[str, Any]) -> dict[str, Any]:
payload = row.get("payload")
if payload is not None and not isinstance(payload, dict):
payload = json.loads(payload) if isinstance(payload, str) else payload
return {
"id": str(row["id"]),
"channel": row["channel"],
"kind": row["kind"],
"payload": payload or {},
"delivered": row["delivered"],
"error": row.get("error"),
"created_at": row["created_at"].isoformat() if row.get("created_at") else None,
}
class LogChannel:
"""Writes notification entries to notification_log with delivered=true."""
channel_name: str = "log"
def send(self, kind: str, text: str, data: dict[str, Any]) -> bool:
row = execute(
"""
INSERT INTO notification_log (channel, kind, payload, delivered, error)
VALUES (%s, %s, %s, true, NULL)
RETURNING *
""",
(self.channel_name, kind, json.dumps({"text": text, **data})),
)
if row is None:
logger.error("LogChannel: insert notification_log failed")
return False
logger.info("LogChannel: delivered kind=%s", kind)
return True
class WebhookChannel:
"""POSTs notification payload to NOTIFY_WEBHOOK_URL.
On 2xx response: writes notification_log with delivered=true.
On non-2xx or exception: writes notification_log with delivered=false and error.
"""
channel_name: str = "webhook"
def __init__(self, url: str | None = None) -> None:
self.url = url or os.environ.get("NOTIFY_WEBHOOK_URL", "")
def send(self, kind: str, text: str, data: dict[str, Any]) -> bool:
payload: dict[str, Any] = {"kind": kind, "text": text, "data": data}
error: str | None = None
delivered = False
if not self.url:
error = "NOTIFY_WEBHOOK_URL not configured"
logger.warning("WebhookChannel: %s", error)
else:
try:
import httpx
resp = httpx.post(self.url, json=payload, timeout=10)
if 200 <= resp.status_code < 300:
delivered = True
else:
error = f"HTTP {resp.status_code}: {resp.text[:200]}"
logger.warning("WebhookChannel: %s", error)
except Exception as exc:
error = str(exc)
logger.warning("WebhookChannel: exception: %s", error)
row = execute(
"""
INSERT INTO notification_log (channel, kind, payload, delivered, error)
VALUES (%s, %s, %s, %s, %s)
RETURNING *
""",
(
self.channel_name,
kind,
json.dumps(payload),
delivered,
error,
),
)
if row is None:
logger.error("WebhookChannel: insert notification_log failed")
return delivered
# --- Channel registry ---
_channels: list[NotificationChannel] | None = None
def get_channels() -> list[NotificationChannel]:
"""Return the list of active notification channels.
LogChannel is always included.
WebhookChannel is included when NOTIFY_WEBHOOK_URL is set.
"""
global _channels
if _channels is not None:
return _channels
channels: list[NotificationChannel] = [LogChannel()]
webhook_url = os.environ.get("NOTIFY_WEBHOOK_URL", "").strip()
if webhook_url:
channels.append(WebhookChannel())
_channels = channels
return channels
def set_channels(channels: list[NotificationChannel] | None) -> None:
"""Override channel list (for testing)."""
global _channels
_channels = channels
def reset_channels() -> None:
"""Reset to default (for testing)."""
global _channels
_channels = None
def send_notification(kind: str, text: str, data: dict[str, Any] | None = None) -> None:
"""Send a notification via all active channels.
kind: e.g. 'daily_digest', 'email_suggestion'
text: human-readable notification text
data: structured payload dict
"""
chs = get_channels()
payload = data or {}
for ch in chs:
try:
ch.send(kind, text, payload)
except Exception:
logger.exception("send_notification: channel %s failed", type(ch).__name__)
# --- Repository helpers ---
def list_notification_log(limit: int = 50) -> list[dict[str, Any]]:
"""Return recent notification_log rows, newest first."""
rows = fetch_all(
"""
SELECT * FROM notification_log
ORDER BY created_at DESC
LIMIT %s
""",
(limit,),
)
return [_normalize_log_row(r) for r in rows]

View file

@ -1,10 +1,7 @@
"""APScheduler integration: daily fetch + batch score + imap poll + digest.
"""APScheduler integration: daily fetch + batch score job.
Starts during app lifespan when SCHEDULER_ENABLED=true (default false).
Jobs:
- daily_fetch_score at 07:00: fetch postings + batch score
- daily_digest at 07:30: send daily digest notification
- imap_poll every 15 min: poll inbox for new emails (gated by EMAIL_WATCH_ENABLED)
Runs a daily job at 07:00 that fetches postings and batch-scores pending applications.
"""
from __future__ import annotations
@ -26,15 +23,6 @@ def is_scheduler_enabled() -> bool:
)
def is_email_watch_enabled() -> bool:
"""Check if email watch is enabled via env."""
return os.environ.get("EMAIL_WATCH_ENABLED", "false").lower() in (
"true",
"1",
"yes",
)
async def _daily_fetch_and_score() -> None:
"""Daily job: fetch postings and batch-score pending applications."""
logger.info("Scheduler: running daily fetch + batch score")
@ -68,67 +56,6 @@ async def _daily_fetch_and_score() -> None:
logger.exception("Scheduler: daily job failed")
async def _daily_digest() -> None:
"""Daily digest: build /today payload text and send notification."""
logger.info("Scheduler: running daily digest")
try:
from app.notify import send_notification
from app.db import repo_app
digest_apps = repo_app.get_digest(limit=20)
nudge_apps = repo_app.get_nudge_applications()
pending = repo_app.count_pending_approvals()
lines: list[str] = []
lines.append("Daily Digest")
lines.append(f"Scored applications: {len(digest_apps)}")
if digest_apps:
lines.append("")
lines.append("Top opportunities:")
for item in digest_apps[:5]:
score = item.get("score")
score_str = f" (score: {int(score)})" if score else ""
lines.append(
f" - {item.get('title', '?')} at {item.get('company', '?')}{score_str}"
)
if nudge_apps:
lines.append("")
lines.append(f"Follow-up nudges: {len(nudge_apps)}")
for n in nudge_apps[:5]:
lines.append(
f" - {n.get('title', '?')} at {n.get('company', '?')}"
f" ({n.get('days_since_sent', 0)} days since sent)"
)
if pending:
lines.append("")
lines.append(f"Pending approvals: {pending}")
text = "\n".join(lines)
send_notification("daily_digest", text, {
"digest_count": len(digest_apps),
"nudge_count": len(nudge_apps),
"pending_approvals": pending,
})
logger.info("Scheduler: daily digest sent")
except Exception:
logger.exception("Scheduler: daily digest failed")
async def _imap_poll() -> None:
"""IMAP poll job: check for new emails and create suggestions."""
if not is_email_watch_enabled():
return
logger.info("Scheduler: running imap poll")
try:
from app.imap_watch import poll_inbox
created = poll_inbox()
logger.info("Scheduler: imap poll created %s suggestions", len(created))
except Exception:
logger.exception("Scheduler: imap poll failed")
def start_scheduler() -> None:
"""Start the APScheduler if enabled."""
global _scheduler
@ -139,7 +66,6 @@ def start_scheduler() -> None:
try:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
except ImportError:
logger.warning(
"APScheduler not installed; scheduler will not start."
@ -153,20 +79,8 @@ def start_scheduler() -> None:
id="daily_fetch_score",
replace_existing=True,
)
_scheduler.add_job(
_daily_digest,
CronTrigger(hour=7, minute=30),
id="daily_digest",
replace_existing=True,
)
_scheduler.add_job(
_imap_poll,
IntervalTrigger(minutes=15),
id="imap_poll",
replace_existing=True,
)
_scheduler.start()
logger.info("Scheduler started: daily fetch+score at 07:00, digest at 07:30, imap poll every 15 min")
logger.info("Scheduler started: daily fetch+score at 07:00")
def stop_scheduler() -> None:

View file

@ -319,31 +319,4 @@ class SeedDemoResponse(BaseModel):
profile: str
postings: int
applications: int
sections: int
# --- v1.1: Email Suggestions ---
class EmailSuggestionOut(BaseModel):
id: str
application_id: str | None = None
mailbox_from: str
subject: str
snippet: str
classification: str
state_proposal: str | None = None
status: str
received_at: str | None = None
created_at: str | None = None
# --- v1.1: Notification Log ---
class NotificationLogOut(BaseModel):
id: str
channel: str
kind: str
payload: dict[str, Any] = {}
delivered: bool
error: str | None = None
created_at: str | None = None
sections: int

View file

@ -1,24 +0,0 @@
-- 003_email_notify.sql -- email suggestions + notification log
CREATE TABLE IF NOT EXISTS email_suggestion (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
application_id uuid REFERENCES application(id) ON DELETE SET NULL,
mailbox_from text NOT NULL,
subject text NOT NULL,
snippet text NOT NULL,
classification text NOT NULL CHECK (classification IN ('interview_invite','rejection','question','noise')),
state_proposal text,
status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','accepted','dismissed')),
received_at timestamptz NOT NULL,
created_at timestamptz DEFAULT now()
);
CREATE TABLE IF NOT EXISTS notification_log (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
channel text NOT NULL,
kind text NOT NULL,
payload jsonb NOT NULL,
delivered boolean NOT NULL,
error text,
created_at timestamptz DEFAULT now()
);

View file

@ -38,7 +38,7 @@ def _truncate_tables():
with psycopg.connect(DATABASE_URL) as conn:
conn.execute(
"""
TRUNCATE TABLE notification_log, email_suggestion, task_run, outbox, approval, artifact,
TRUNCATE TABLE task_run, outbox, approval, artifact,
application, job_posting, cv_section, profile
RESTART IDENTITY CASCADE
"""

View file

@ -1,682 +0,0 @@
"""Tests for v1.1: email watch, notifications, suggestions endpoints.
Covers:
- IMAP matching logic (company in subject, sender domain in URL, title keywords)
- Classifier -> suggestion row
- Noise dedupe (same from+subject+day)
- Accept applies transition through guard
- Dismiss marks suggestion
- Webhook success/failure notification_log rows
- LogChannel writes delivered=true
- Daily digest payload shape
- GET /suggestions, POST accept, POST dismiss, GET /notifications/log
- FakeImap end-to-end poll
"""
from __future__ import annotations
import email as email_mod
from datetime import datetime, timedelta, timezone
import pytest
from fastapi.testclient import TestClient
from app.config import DATABASE_URL
from app.db import repo_app
from app.imap_watch import (
FakeImap,
_build_snippet,
_extract_sender_domain,
_parse_email_message,
classify_email,
create_email_suggestion,
is_duplicate,
match_application,
poll_inbox,
)
from app.notify import (
LogChannel,
WebhookChannel,
get_channels,
list_notification_log,
reset_channels,
send_notification,
set_channels,
)
@pytest.fixture()
def client():
from app.main import app
return TestClient(app)
# --- Helpers ---
def _make_email(raw_from: str, subject: str, body: str, date: str = "") -> bytes:
"""Build raw email bytes for FakeImap."""
msg = email_mod.message_from_string(
f"From: {raw_from}\r\n"
f"Subject: {subject}\r\n"
f"Date: {date or 'Mon, 01 Jul 2026 10:00:00 +0000'}\r\n"
f"\r\n"
f"{body}"
)
return msg.as_bytes()
def _create_app_in_state(state: str = "sent", company: str = "TechCorp", url: str = "https://techcorp.com/jobs/1") -> dict:
"""Create a posting + application, force state via SQL."""
posting = repo_app.create_job_posting(
source="manual_url",
url=url,
company=company,
title="Senior Python Developer",
location="Malmo",
description="",
raw={},
)
app_row = repo_app.create_application(posting["id"])
if state != "discovered":
repo_app.update_application_score(app_row["id"], 80, {"factors": {}})
if state in ("approved", "rejected"):
repo_app.update_application_state(app_row["id"], "scored")
repo_app.update_application_state(app_row["id"], state)
elif state == "sent":
repo_app.update_application_state(app_row["id"], "scored")
repo_app.update_application_state(app_row["id"], "approved")
repo_app.update_application_state(app_row["id"], "drafting")
# Bypass guard for test
from app.db import execute
execute(
"UPDATE application SET state = 'sent', last_activity_at = now() WHERE id = %s",
(app_row["id"],),
)
elif state == "interviewing":
repo_app.update_application_state(app_row["id"], "scored")
repo_app.update_application_state(app_row["id"], "approved")
repo_app.update_application_state(app_row["id"], "drafting")
from app.db import execute
execute(
"UPDATE application SET state = 'sent' WHERE id = %s",
(app_row["id"],),
)
execute(
"UPDATE application SET state = 'interviewing' WHERE id = %s",
(app_row["id"],),
)
return app_row
# ========================================================================
# IMAP matching logic (3 tests)
# ========================================================================
class TestImapMatching:
def test_match_by_company_name_in_subject(self):
"""Email subject contains company name -> match."""
apps = [
{"id": "app1", "state": "sent", "company": "TechCorp", "title": "Python Dev", "url": "https://example.com"},
]
result = match_application(
"recruiter@gmail.com",
"Interview at TechCorp",
"Please come for an interview.",
apps,
)
assert result is not None
assert result["id"] == "app1"
def test_match_by_sender_domain_in_url(self):
"""Sender domain matches the posting URL -> match."""
apps = [
{"id": "app2", "state": "sent", "company": "Unknown", "title": "Dev", "url": "https://techcorp.com/careers/1"},
]
result = match_application(
"hr@techcorp.com",
"Your application",
"We reviewed your application.",
apps,
)
assert result is not None
assert result["id"] == "app2"
def test_match_by_title_keywords(self):
"""Email subject contains 2+ title words -> match."""
apps = [
{"id": "app3", "state": "interviewing", "company": "SomeCompany", "title": "Senior Python Developer", "url": "https://other.com"},
]
result = match_application(
"someone@other.com",
"Senior Python position update",
"Regarding the developer role.",
apps,
)
assert result is not None
assert result["id"] == "app3"
def test_no_match_wrong_state(self):
"""Applications in discovered state are not matched."""
apps = [
{"id": "app4", "state": "discovered", "company": "TechCorp", "title": "Dev", "url": "https://techcorp.com"},
]
result = match_application(
"hr@techcorp.com",
"Interview at TechCorp",
"Come for an interview.",
apps,
)
assert result is None
def test_no_match_unrelated_email(self):
"""Email unrelated to any application -> no match."""
apps = [
{"id": "app5", "state": "sent", "company": "TechCorp", "title": "Dev", "url": "https://techcorp.com"},
]
result = match_application(
"newsletter@spam.com",
"Buy now!",
"Special offer for you.",
apps,
)
assert result is None
# ========================================================================
# Classifier -> row (3 tests)
# ========================================================================
class TestClassifierToRow:
def test_classify_returns_interview_invite(self):
"""classify_email returns interview_invite from mock."""
result = classify_email("Interview invitation", "Please come for an interview next week.")
assert result["classification"] == "interview_invite"
assert result["state_proposal"] == "interviewing"
assert "reason" in result
def test_classify_falls_back_on_invalid_classification(self):
"""Invalid classification from LLM falls back to noise."""
import app.llm as llm_mod
original = llm_mod.MOCK_OUTPUTS.get("email_classify", {}).copy()
try:
llm_mod.MOCK_OUTPUTS["email_classify"] = {"classification": "bogus", "state_proposal": None, "reason": "test"}
result = classify_email("test", "test")
assert result["classification"] == "noise"
finally:
llm_mod.MOCK_OUTPUTS["email_classify"] = original
def test_create_email_suggestion_row(self):
"""create_email_suggestion inserts a row correctly."""
now = datetime.now(timezone.utc)
suggestion = create_email_suggestion(
application_id=None,
mailbox_from="hr@example.com",
subject="Interview invite",
snippet="Please come for an interview.",
classification="interview_invite",
state_proposal="interviewing",
received_at=now,
)
assert suggestion["id"] is not None
assert suggestion["mailbox_from"] == "hr@example.com"
assert suggestion["classification"] == "interview_invite"
assert suggestion["status"] == "pending"
assert suggestion["state_proposal"] == "interviewing"
# ========================================================================
# Noise dedupe (3 tests)
# ========================================================================
class TestNoiseDedupe:
def test_duplicate_detected_same_day(self):
"""Same from+subject+day is flagged as duplicate."""
now = datetime.now(timezone.utc)
create_email_suggestion(
application_id=None,
mailbox_from="hr@example.com",
subject="Interview",
snippet="Come for an interview.",
classification="interview_invite",
state_proposal="interviewing",
received_at=now,
)
assert is_duplicate("hr@example.com", "Interview", now)
def test_different_subject_not_duplicate(self):
"""Different subject -> not duplicate."""
now = datetime.now(timezone.utc)
create_email_suggestion(
application_id=None,
mailbox_from="hr@example.com",
subject="Interview",
snippet="Come.",
classification="interview_invite",
state_proposal="interviewing",
received_at=now,
)
assert not is_duplicate("hr@example.com", "Different Subject", now)
def test_different_sender_not_duplicate(self):
"""Different sender -> not duplicate."""
now = datetime.now(timezone.utc)
create_email_suggestion(
application_id=None,
mailbox_from="hr@example.com",
subject="Interview",
snippet="Come.",
classification="interview_invite",
state_proposal="interviewing",
received_at=now,
)
assert not is_duplicate("other@example.com", "Interview", now)
# ========================================================================
# Accept applies transition through guard (3 tests)
# ========================================================================
class TestAcceptSuggestion:
def test_accept_applies_transition(self, client):
"""POST /suggestions/{id}/accept transitions app from sent to interviewing."""
app_row = _create_app_in_state("sent", company="TechCorp")
suggestion = create_email_suggestion(
application_id=app_row["id"],
mailbox_from="hr@techcorp.com",
subject="Interview at TechCorp",
snippet="Please come in for an interview.",
classification="interview_invite",
state_proposal="interviewing",
received_at=datetime.now(timezone.utc),
)
resp = client.post(f"/api/suggestions/{suggestion['id']}/accept")
assert resp.status_code == 200
assert resp.json()["status"] == "accepted"
# Verify application state changed
updated_app = repo_app.get_application(app_row["id"])
assert updated_app["state"] == "interviewing"
def test_accept_invalid_transition_409(self, client):
"""Accept with invalid transition (e.g. discovered -> interviewing) returns 409."""
posting = repo_app.create_job_posting(
source="manual_url", url="https://example.com/bad/1",
company="X", title="X", location=None, description="", raw={},
)
app_row = repo_app.create_application(posting["id"])
suggestion = create_email_suggestion(
application_id=app_row["id"],
mailbox_from="hr@example.com",
subject="Test",
snippet="Test",
classification="interview_invite",
state_proposal="interviewing",
received_at=datetime.now(timezone.utc),
)
resp = client.post(f"/api/suggestions/{suggestion['id']}/accept")
assert resp.status_code == 409
assert "invalid_transition" in str(resp.json()["detail"])
def test_accept_404_nonexistent(self, client):
"""Accept on nonexistent suggestion -> 404."""
resp = client.post("/api/suggestions/00000000-0000-0000-0000-000000000000/accept")
assert resp.status_code == 404
def test_accept_already_accepted_409(self, client):
"""Accept on already accepted suggestion -> 409."""
app_row = _create_app_in_state("sent")
suggestion = create_email_suggestion(
application_id=app_row["id"],
mailbox_from="hr@example.com",
subject="Test",
snippet="Test",
classification="interview_invite",
state_proposal="interviewing",
received_at=datetime.now(timezone.utc),
)
client.post(f"/api/suggestions/{suggestion['id']}/accept")
resp = client.post(f"/api/suggestions/{suggestion['id']}/accept")
assert resp.status_code == 409
# ========================================================================
# Dismiss suggestion (2 tests)
# ========================================================================
class TestDismissSuggestion:
def test_dismiss_marks_as_dismissed(self, client):
"""POST /suggestions/{id}/dismiss marks as dismissed."""
suggestion = create_email_suggestion(
application_id=None,
mailbox_from="hr@example.com",
subject="Test",
snippet="Test",
classification="question",
state_proposal=None,
received_at=datetime.now(timezone.utc),
)
resp = client.post(f"/api/suggestions/{suggestion['id']}/dismiss")
assert resp.status_code == 200
assert resp.json()["status"] == "dismissed"
def test_dismiss_404_nonexistent(self, client):
"""Dismiss nonexistent -> 404."""
resp = client.post("/api/suggestions/00000000-0000-0000-0000-000000000000/dismiss")
assert resp.status_code == 404
# ========================================================================
# GET /suggestions (2 tests)
# ========================================================================
class TestGetSuggestions:
def test_get_suggestions_returns_pending(self, client):
"""GET /suggestions returns only pending suggestions."""
create_email_suggestion(
application_id=None,
mailbox_from="a@example.com",
subject="Subject A",
snippet="Snippet A",
classification="interview_invite",
state_proposal="interviewing",
received_at=datetime.now(timezone.utc),
)
create_email_suggestion(
application_id=None,
mailbox_from="b@example.com",
subject="Subject B",
snippet="Snippet B",
classification="rejection",
state_proposal=None,
received_at=datetime.now(timezone.utc),
)
resp = client.get("/api/suggestions")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert all(s["status"] == "pending" for s in data)
def test_get_suggestions_empty(self, client):
"""GET /suggestions returns empty list when no suggestions."""
resp = client.get("/api/suggestions")
assert resp.status_code == 200
assert resp.json() == []
# ========================================================================
# Notification channels (6 tests)
# ========================================================================
class TestLogChannel:
def test_log_channel_writes_delivered_true(self):
"""LogChannel writes notification_log with delivered=true."""
reset_channels()
ch = LogChannel()
result = ch.send("daily_digest", "Test digest", {"count": 5})
assert result is True
logs = list_notification_log(limit=10)
assert len(logs) >= 1
latest = logs[0]
assert latest["channel"] == "log"
assert latest["kind"] == "daily_digest"
assert latest["delivered"] is True
assert latest["error"] is None
def test_send_notification_log_channel(self):
"""send_notification via LogChannel creates a log entry."""
reset_channels()
set_channels([LogChannel()])
send_notification("email_suggestion", "Interview invite from TechCorp", {"id": "test"})
logs = list_notification_log(limit=10)
assert len(logs) >= 1
assert logs[0]["kind"] == "email_suggestion"
assert logs[0]["delivered"] is True
reset_channels()
class TestWebhookChannel:
def test_webhook_success_2xx(self, monkeypatch):
"""WebhookChannel with 2xx response writes delivered=true."""
class FakeResponse:
status_code = 200
text = "OK"
class FakeClient:
@staticmethod
def post(url, json=None, timeout=None):
assert url == "https://hook.example.com/notify"
return FakeResponse()
import httpx
monkeypatch.setattr(httpx, "post", FakeClient.post)
reset_channels()
ch = WebhookChannel(url="https://hook.example.com/notify")
result = ch.send("daily_digest", "Digest", {"count": 3})
assert result is True
logs = list_notification_log(limit=10)
latest = logs[0]
assert latest["channel"] == "webhook"
assert latest["delivered"] is True
assert latest["error"] is None
reset_channels()
def test_webhook_failure_non_2xx(self, monkeypatch):
"""WebhookChannel with non-2xx writes delivered=false with error."""
class FakeResponse:
status_code = 500
text = "Internal Server Error"
class FakeClient:
@staticmethod
def post(url, json=None, timeout=None):
return FakeResponse()
import httpx
monkeypatch.setattr(httpx, "post", FakeClient.post)
reset_channels()
ch = WebhookChannel(url="https://hook.example.com/notify")
result = ch.send("daily_digest", "Digest", {"count": 3})
assert result is False
logs = list_notification_log(limit=10)
latest = logs[0]
assert latest["channel"] == "webhook"
assert latest["delivered"] is False
assert latest["error"] is not None
assert "500" in latest["error"]
reset_channels()
def test_webhook_exception_writes_error(self, monkeypatch):
"""WebhookChannel with connection exception writes delivered=false."""
def raise_exc(url, json=None, timeout=None):
raise ConnectionError("Connection refused")
import httpx
monkeypatch.setattr(httpx, "post", raise_exc)
reset_channels()
ch = WebhookChannel(url="https://hook.example.com/notify")
result = ch.send("daily_digest", "Digest", {"count": 1})
assert result is False
logs = list_notification_log(limit=10)
latest = logs[0]
assert latest["delivered"] is False
assert "Connection refused" in (latest["error"] or "")
reset_channels()
def test_webhook_no_url_writes_error(self):
"""WebhookChannel with no URL writes delivered=false with config error."""
reset_channels()
ch = WebhookChannel(url="")
result = ch.send("test", "test", {})
assert result is False
logs = list_notification_log(limit=10)
latest = logs[0]
assert latest["delivered"] is False
assert "not configured" in (latest["error"] or "")
reset_channels()
# ========================================================================
# Notification log endpoint (2 tests)
# ========================================================================
class TestNotificationLogEndpoint:
def test_get_notifications_log(self, client):
"""GET /notifications/log returns entries."""
reset_channels()
set_channels([LogChannel()])
send_notification("daily_digest", "Test", {"count": 1})
reset_channels()
resp = client.get("/api/notifications/log")
assert resp.status_code == 200
data = resp.json()
assert len(data) >= 1
assert "channel" in data[0]
assert "kind" in data[0]
assert "delivered" in data[0]
def test_get_notifications_log_empty(self, client):
"""GET /notifications/log returns empty when no entries."""
resp = client.get("/api/notifications/log")
assert resp.status_code == 200
assert resp.json() == []
# ========================================================================
# Digest payload shape (2 tests)
# ========================================================================
class TestDigestPayload:
def test_daily_digest_notification_text(self):
"""Daily digest notification contains expected text fields."""
reset_channels()
set_channels([LogChannel()])
send_notification("daily_digest", "Daily Digest\nScored applications: 5", {
"digest_count": 5,
"nudge_count": 2,
"pending_approvals": 1,
})
logs = list_notification_log(limit=10)
entry = logs[0]
assert entry["kind"] == "daily_digest"
payload = entry["payload"]
assert "text" in payload
assert "Daily Digest" in payload["text"]
assert payload.get("digest_count") == 5
reset_channels()
def test_daily_digest_payload_has_counts(self):
"""Digest payload includes digest_count, nudge_count, pending_approvals."""
reset_channels()
set_channels([LogChannel()])
send_notification("daily_digest", "text", {
"digest_count": 3,
"nudge_count": 1,
"pending_approvals": 0,
})
logs = list_notification_log(limit=10)
payload = logs[0]["payload"]
assert payload["digest_count"] == 3
assert payload["nudge_count"] == 1
assert payload["pending_approvals"] == 0
reset_channels()
# ========================================================================
# FakeImap end-to-end poll (3 tests)
# ========================================================================
class TestFakeImapPoll:
def test_poll_creates_suggestion_for_matching_email(self):
"""FakeImap poll creates a suggestion when email matches an application."""
app_row = _create_app_in_state("sent", company="TechCorp", url="https://techcorp.com/jobs/1")
raw_email = _make_email(
"hr@techcorp.com",
"Interview at TechCorp",
"Please come for an interview next Tuesday.",
)
fake = FakeImap(messages=[(b"1", raw_email)])
created = poll_inbox(fake)
assert len(created) == 1
assert created[0]["classification"] == "interview_invite"
assert created[0]["mailbox_from"] == "hr@techcorp.com"
assert created[0]["application_id"] == app_row["id"]
def test_poll_skips_noise_emails(self):
"""FakeImap poll skips noise classification (no suggestion created)."""
_create_app_in_state("sent", company="TechCorp")
# Mock email_classify to return noise
import app.llm as llm_mod
original = llm_mod.MOCK_OUTPUTS.get("email_classify", {}).copy()
try:
llm_mod.MOCK_OUTPUTS["email_classify"] = {
"classification": "noise",
"state_proposal": None,
"reason": "spam",
}
raw_email = _make_email(
"newsletter@spam.com",
"Buy our product",
"Special offer just for you!",
)
fake = FakeImap(messages=[(b"1", raw_email)])
created = poll_inbox(fake)
assert len(created) == 0
finally:
llm_mod.MOCK_OUTPUTS["email_classify"] = original
def test_poll_dedupe_skips_same_from_subject_day(self):
"""FakeImap poll dedupes same from+subject+day."""
raw_email = _make_email(
"hr@example.com",
"Same Subject",
"Same body content.",
)
# First poll
fake1 = FakeImap(messages=[(b"1", raw_email)])
created1 = poll_inbox(fake1)
assert len(created1) == 1
# Second poll with same message -> dedupe
fake2 = FakeImap(messages=[(b"1", raw_email)])
created2 = poll_inbox(fake2)
assert len(created2) == 0 # deduped
# ========================================================================
# Email parsing helpers (2 tests)
# ========================================================================
class TestEmailParsing:
def test_extract_sender_domain(self):
"""Extract domain from From header."""
assert _extract_sender_domain("John Doe <hr@techcorp.com>") == "techcorp.com"
assert _extract_sender_domain("noreply@example.org") == "example.org"
assert _extract_sender_domain("") == ""
def test_build_snippet_truncates(self):
"""Snippet is truncated to max_len."""
long_body = "A" * 500
snippet = _build_snippet(long_body, max_len=50)
assert len(snippet) <= 53 # 50 + "..."
assert snippet.endswith("...")
def test_build_snippet_short_body(self):
"""Short body is not truncated."""
snippet = _build_snippet("Hello", max_len=300)
assert snippet == "Hello"

View file

@ -30,12 +30,9 @@ TASK_CLASS_MAP: dict[str, TaskClass] = {
"score": TaskClass.CHEAP,
"extract": TaskClass.CHEAP,
"cv_assist": TaskClass.CHEAP,
"email_classify": TaskClass.CHEAP,
"deadline_extract": TaskClass.CHEAP,
"cl_critique": TaskClass.STRONG,
"critique": TaskClass.STRONG,
"research": TaskClass.STRONG,
"cv_tailor": TaskClass.STRONG,
}
# Default budgets (max output tokens) per task name.
@ -43,12 +40,9 @@ DEFAULT_BUDGETS: dict[str, int] = {
"score": 2000,
"extract": 4000,
"cv_assist": 2000,
"email_classify": 1000,
"deadline_extract": 500,
"cl_critique": 4000,
"critique": 6000,
"research": 4000,
"cv_tailor": 6000,
}

View file

@ -59,41 +59,6 @@ MOCK_OUTPUTS: dict[str, dict] = {
"summary": "The company is a mid-size tech firm focused on cloud infrastructure.",
"key_points": ["Founded in 2015", "Series B funding", "Remote-first culture"],
},
"email_classify": {
"classification": "interview_invite",
"state_proposal": "interviewing",
"reason": "The email contains an invitation to schedule an interview.",
},
"cv_tailor": {
"tailored_cv": {
"summary": "Senior Python Developer with 6+ years building scalable backend systems.",
"skills": [
"Python",
"Fast API",
"PostgreSQL",
"Docker",
"Kubernetes",
"AWS",
],
"experience": [
{
"company": "TechCorp",
"role": "Senior Backend Engineer",
"bullets": [
"Led migration of monolith to microservices using Fast API",
"Reduced API latency by 40% through query optimization and caching",
],
},
],
},
"change_log": [
{"action": "reordered", "detail": "Moved Python and Fast API to top of skills"},
{"action": "rephrased", "detail": "Rewrote first experience bullet to emphasize Fast API"},
],
},
"deadline_extract": {
"apply_by": None,
},
}
# Default mock output for unknown task names.

View file

@ -270,11 +270,8 @@ class TestGatewayConfig:
assert config.get_task_class("score") == TaskClass.CHEAP
assert config.get_task_class("extract") == TaskClass.CHEAP
assert config.get_task_class("cv_assist") == TaskClass.CHEAP
assert config.get_task_class("email_classify") == TaskClass.CHEAP
assert config.get_task_class("deadline_extract") == TaskClass.CHEAP
assert config.get_task_class("critique") == TaskClass.STRONG
assert config.get_task_class("cl_critique") == TaskClass.STRONG
assert config.get_task_class("cv_tailor") == TaskClass.STRONG
def test_get_model_routing(self) -> None:
config = mock_config(cheap_model="cheap-model", strong_model="strong-model")

View file

@ -1,125 +0,0 @@
"""Tests for new v1.1 mock tasks: email_classify, cv_tailor, deadline_extract."""
from __future__ import annotations
import pytest
from llm_gateway.config import GatewayConfig, ProviderConfig, TaskClass
from llm_gateway.gateway import Gateway
from llm_gateway.mock import get_mock_output, MOCK_OUTPUTS
def mock_config(**overrides) -> GatewayConfig:
"""Build a config in mock mode (no API key)."""
primary = ProviderConfig(
name="primary",
base_url="https://mock.example.com/v1",
api_key="",
model="glm-5.2",
)
defaults = {
"primary": primary,
"fallback": None,
"cheap_model": "glm-5.2",
"strong_model": "glm-5.2",
"budgets": {
"score": 2000,
"extract": 4000,
"email_classify": 1000,
"deadline_extract": 500,
"cv_tailor": 6000,
"default": 4000,
},
"max_retries": 2,
}
defaults.update(overrides)
return GatewayConfig(**defaults)
class TestEmailClassifyMock:
async def test_email_classify_returns_deterministic(self) -> None:
"""email_classify mock returns interview_invite classification."""
config = mock_config()
gw = Gateway(config)
result_a = await gw.run_task("email_classify", "Email from recruiter")
result_b = await gw.run_task("email_classify", "Email from recruiter")
assert result_a == result_b
assert result_a["classification"] == "interview_invite"
assert result_a["state_proposal"] == "interviewing"
assert "reason" in result_a
await gw.aclose()
async def test_email_classify_is_cheap(self) -> None:
"""email_classify should be classified as CHEAP."""
config = mock_config()
assert config.get_task_class("email_classify") == TaskClass.CHEAP
assert config.get_model("email_classify") == config.cheap_model
def test_email_classify_in_mock_outputs(self) -> None:
"""email_classify should be in MOCK_OUTPUTS."""
assert "email_classify" in MOCK_OUTPUTS
output = get_mock_output("email_classify")
assert output["classification"] == "interview_invite"
assert output["state_proposal"] == "interviewing"
class TestCvTailorMock:
async def test_cv_tailor_returns_deterministic(self) -> None:
"""cv_tailor mock returns tailored CV with change_log."""
config = mock_config()
gw = Gateway(config)
result_a = await gw.run_task("cv_tailor", "Tailor CV for posting")
result_b = await gw.run_task("cv_tailor", "Tailor CV for posting")
assert result_a == result_b
assert "tailored_cv" in result_a
assert "change_log" in result_a
assert isinstance(result_a["change_log"], list)
assert len(result_a["change_log"]) >= 1
# Check change_log entries have action and detail.
for entry in result_a["change_log"]:
assert "action" in entry
assert "detail" in entry
await gw.aclose()
async def test_cv_tailor_is_strong(self) -> None:
"""cv_tailor should be classified as STRONG."""
config = mock_config()
assert config.get_task_class("cv_tailor") == TaskClass.STRONG
assert config.get_model("cv_tailor") == config.strong_model
def test_cv_tailor_in_mock_outputs(self) -> None:
"""cv_tailor should be in MOCK_OUTPUTS."""
assert "cv_tailor" in MOCK_OUTPUTS
output = get_mock_output("cv_tailor")
assert "tailored_cv" in output
assert "change_log" in output
# Check it has skills and experience.
assert "skills" in output["tailored_cv"]
assert "experience" in output["tailored_cv"]
class TestDeadlineExtractMock:
async def test_deadline_extract_returns_deterministic(self) -> None:
"""deadline_extract mock returns apply_by (null by default)."""
config = mock_config()
gw = Gateway(config)
result_a = await gw.run_task("deadline_extract", "Extract deadline")
result_b = await gw.run_task("deadline_extract", "Extract deadline")
assert result_a == result_b
assert "apply_by" in result_a
# Default mock has null deadline.
assert result_a["apply_by"] is None
await gw.aclose()
async def test_deadline_extract_is_cheap(self) -> None:
"""deadline_extract should be classified as CHEAP."""
config = mock_config()
assert config.get_task_class("deadline_extract") == TaskClass.CHEAP
assert config.get_model("deadline_extract") == config.cheap_model
def test_deadline_extract_in_mock_outputs(self) -> None:
"""deadline_extract should be in MOCK_OUTPUTS."""
assert "deadline_extract" in MOCK_OUTPUTS
output = get_mock_output("deadline_extract")
assert "apply_by" in output
assert output["apply_by"] is None

View file

@ -1,40 +0,0 @@
# packages/matching
Job posting similarity, dedupe clustering, and keyword coverage for the
jobhunt-platform v1.1 agency duplicate detection feature (ADR-0003).
## Modules
### similarity.py
- `normalize_employer(name)` -- lowercase, strip agency/legal suffixes (AB, Consulting, etc.), remove punctuation.
- `title_score(a, b)` -- rapidfuzz token_set_ratio on job titles (0-100).
- `employer_match(a, b)` -- True if normalized employer names are equal.
- `desc_score(a, b, max_chars=2000)` -- token_set_ratio on first 2000 chars of descriptions.
### dedupe.py
- `cluster(postings: list[dict]) -> dict[str, list[str]]` -- group postings into duplicate clusters.
Clustering rule (per ADR-0003):
- Same employer (normalized) **OR**
- Title similarity >= 85 **AND** description similarity >= 80
Uses union-find for transitive grouping. Clusters are sorted by descending
max pairwise score (`c1` = tightest cluster).
### keywords.py
- `extract_keywords(text, top_n=30)` -- frequency-based keyword extraction with Swedish + English stopword removal.
- `coverage(cv_text, posting_text)` -- computes keyword coverage of a CV against a job posting.
Multiword tech terms like "fast api" are collapsed to "fastapi" so they survive as single keywords.
## Installation (uv)
```bash
cd packages/matching
uv venv && source .venv/bin/activate
uv pip install -e ".[dev]"
pytest
```
## License
MIT

View file

@ -1,26 +0,0 @@
[project]
name = "matching"
version = "0.1.0"
description = "Job posting similarity, dedupe clustering, and keyword coverage for agency duplicate detection."
requires-python = ">=3.13"
dependencies = [
"rapidfuzz>=3.6",
]
[project.optional-dependencies]
dev = [
"pytest>=8.0",
"pytest-asyncio>=0.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/matching"]
[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = ["src"]
asyncio_mode = "auto"

View file

@ -1,28 +0,0 @@
"""Job posting matching package.
Provides:
- similarity: normalized text comparison (employer match, title/desc scores)
- dedupe: cluster postings into duplicate groups
- keywords: keyword extraction and CV-vs-posting coverage
"""
from __future__ import annotations
from matching.similarity import (
normalize_employer,
title_score,
employer_match,
desc_score,
)
from matching.dedupe import cluster
from matching.keywords import extract_keywords, coverage
__all__ = [
"normalize_employer",
"title_score",
"employer_match",
"desc_score",
"cluster",
"extract_keywords",
"coverage",
]

View file

@ -1,136 +0,0 @@
"""Duplicate clustering for job postings.
Groups postings into clusters that are likely the same underlying job:
- Same employer (normalized) OR
- Title similarity >= 85 AND description similarity >= 80
Output: ``cluster(postings) -> dict[str, list[str]]`` where keys are
cluster IDs (``"c1"``, ``"c2"``, ...) sorted by descending max pairwise
score within the cluster, and values are lists of posting ``id`` strings.
"""
from __future__ import annotations
from matching.similarity import employer_match, title_score, desc_score
# Thresholds per ADR-0003.
TITLE_THRESHOLD = 85.0
DESC_THRESHOLD = 80.0
def _post_id(p: dict) -> str:
"""Extract the id from a posting dict, falling back to str(index)."""
pid = p.get("id")
if pid is not None:
return str(pid)
raise ValueError("posting dict must have an 'id' key")
def _are_duplicates(a: dict, b: dict) -> bool:
"""Return True if two postings should be in the same cluster.
Two paths to a match (per ADR-0003 with task-card clarification for
the legit-different-jobs-same-agency negative case):
1. Same employer (normalized) AND some content overlap
(title >= 85 OR desc >= 80).
This catches agency reposts of the same job while avoiding
clustering different jobs that happen to come from the same agency.
2. Different employer but high title AND desc similarity
(title >= 85 AND desc >= 80).
This catches cross-agency reposts of the same job.
"""
ts = title_score(a.get("title", ""), b.get("title", ""))
ds = desc_score(a.get("description", ""), b.get("description", ""))
same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
if same_employer:
# Same employer + at least one content dimension similar.
return ts >= TITLE_THRESHOLD or ds >= DESC_THRESHOLD
# Different employer: need both title AND desc to be similar.
return ts >= TITLE_THRESHOLD and ds >= DESC_THRESHOLD
def _pair_score(a: dict, b: dict) -> float:
"""Compute a similarity score between two postings for sorting clusters."""
ts = title_score(a.get("title", ""), b.get("title", ""))
ds = desc_score(a.get("description", ""), b.get("description", ""))
same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
if same_employer:
# Employer match: weight title more for tie-breaking.
return 100.0 + ts
return (ts + ds) / 2.0
def cluster(postings: list[dict]) -> dict[str, list[str]]:
"""Cluster job postings into duplicate groups.
Uses union-find so transitive duplicates (A~B, B~C => A~C) are grouped
together.
Args:
postings: list of dicts with keys ``id``, ``employer``, ``title``,
``description``.
Returns:
Dict mapping cluster_id (``"c1"``, ``"c2"``, ...) to a list of
posting id strings. Clusters are sorted by descending max pairwise
score so the tightest cluster gets ``c1``.
"""
n = len(postings)
if n == 0:
return {}
ids = [_post_id(p) for p in postings]
# Union-find (disjoint set).
parent: list[int] = list(range(n))
def find(x: int) -> int:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[ra] = rb
# O(n^2) pairwise comparison.
for i in range(n):
for j in range(i + 1, n):
if _are_duplicates(postings[i], postings[j]):
union(i, j)
# Collect groups.
groups: dict[int, list[int]] = {}
for i in range(n):
root = find(i)
groups.setdefault(root, []).append(i)
# Compute max pairwise score per group for sorting.
group_scores: list[tuple[float, list[int]]] = []
for indices in groups.values():
if len(indices) < 2:
max_sc = 0.0
else:
max_sc = 0.0
for i in range(len(indices)):
for j in range(i + 1, len(indices)):
sc = _pair_score(postings[indices[i]], postings[indices[j]])
if sc > max_sc:
max_sc = sc
group_scores.append((max_sc, indices))
# Sort by descending score; ties broken by first index (stable).
group_scores.sort(key=lambda t: (-t[0], t[1][0]))
# Assign cluster IDs.
result: dict[str, list[str]] = {}
for idx, (_, indices) in enumerate(group_scores, start=1):
result[f"c{idx}"] = [ids[i] for i in indices]
return result

View file

@ -1,162 +0,0 @@
"""Keyword extraction and coverage analysis.
Deterministic, no LLM. Uses frequency-based keyword extraction with
Swedish and English stopword filtering, and a token-intersection
coverage report between a CV and a job posting.
"""
from __future__ import annotations
import re
from collections import Counter
# ---------------------------------------------------------------------------
# Stopwords (Swedish + English). Conservative lists.
# ---------------------------------------------------------------------------
_SWEDISH_STOPWORDS: frozenset[str] = frozenset({
"och", "eller", "som", "att", "den", "det", "de", "vi", "ni", "du",
"han", "hon", "den", "en", "ett", "ar", "har", "var", "var", "inte",
"med", "for", "fran", "till", "pa", "av", "i", "och", "men", "sa",
"när", "", "hur", "alla", "nagon", "nagot", "alla", "manga", "mycket",
"skall", "ska", "kan", "kommer", "blir", "vore", "vill", "borde",
"efter", "under", "over", "bAKom", "inom", "mellan", "genom", "utan",
"mot", "ut", "fran", "frams", "igår", "idag", "imorgon",
})
_ENGLISH_STOPWORDS: frozenset[str] = frozenset({
"a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
"of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
"being", "have", "has", "had", "do", "does", "did", "will", "would",
"should", "could", "may", "might", "must", "can", "this", "that",
"these", "those", "i", "you", "he", "she", "it", "we", "they", "me",
"him", "her", "us", "them", "my", "your", "his", "its", "our", "their",
"what", "which", "who", "whom", "where", "when", "why", "how", "all",
"each", "every", "both", "few", "more", "most", "other", "some", "such",
"no", "nor", "not", "only", "own", "same", "so", "than", "too", "very",
"as", "if", "about", "against", "between", "into", "through", "during",
"before", "after", "above", "below", "up", "down", "out", "off",
"over", "under", "again", "further", "then", "once", "here", "there",
"also", "etc", "e.g", "i.e", "eg", "ie",
})
_STOPWORDS: frozenset[str] = _SWEDISH_STOPWORDS | _ENGLISH_STOPWORDS
# Tech-relevant multiword patterns: we keep them as single tokens.
# e.g. "fast api" -> "fastapi" so it survives as a keyword.
_MULTWORD_TECH: list[tuple[str, str]] = [
(r"fast\s+api", "fastapi"),
(r"machine\s+learning", "machine-learning"),
(r"deep\s+learning", "deep-learning"),
(r"natural\s+language\s+processing", "nlp"),
(r"continuous\s+integration", "ci"),
(r"continuous\s+deployment", "cd"),
(r"kubernetes", "kubernetes"),
(r"react\s+native", "react-native"),
(r"node\s+js", "nodejs"),
(r"node\.js", "nodejs"),
(r"aws", "aws"),
(r"gcp", "gcp"),
(r"ci/cd", "ci-cd"),
]
# Minimum token length for keywords.
_MIN_TOKEN_LEN = 2
# Tokenizer: split on non-alphanumeric.
_TOKEN_RE = re.compile(r"[a-z0-9+#.\-/]+")
def _preprocess_multiwords(text: str) -> str:
"""Replace known multiword tech terms with single tokens."""
result = text.lower()
for pattern, replacement in _MULTWORD_TECH:
result = re.sub(pattern, replacement, result, flags=re.IGNORECASE)
return result
def _tokenize(text: str) -> list[str]:
"""Tokenize text into lowercased tokens."""
text = _preprocess_multiwords(text)
raw_tokens = _TOKEN_RE.findall(text.lower())
tokens: list[str] = []
for t in raw_tokens:
t = t.strip(".-/")
if not t:
continue
if t in _STOPWORDS:
continue
if len(t) < _MIN_TOKEN_LEN:
continue
# Skip pure numbers (unless they look like versions).
if t.isdigit() and len(t) > 4:
continue
tokens.append(t)
return tokens
def extract_keywords(text: str, top_n: int = 30) -> list[str]:
"""Extract the top *top_n* keywords from *text* by frequency.
Keywords are lowercased tokens. Stopwords (Swedish + English) are
removed. Multiword tech terms like "fast api" are collapsed to
"fastapi".
Args:
text: the text to analyze.
top_n: maximum number of keywords to return.
Returns:
List of keyword strings, most frequent first. Ties are broken
alphabetically for determinism.
"""
if not text:
return []
tokens = _tokenize(text)
if not tokens:
return []
counts: Counter[str] = Counter(tokens)
# Sort by count desc, then alphabetically for deterministic order.
ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
return [word for word, _ in ranked[:top_n]]
def coverage(cv_text: str, posting_text: str) -> dict[str, list[str] | float]:
"""Compute keyword coverage of a CV against a job posting.
Extracts keywords from *posting_text*, extracts keywords from
*cv_text*, and reports which posting keywords are matched in the
CV, which are missing, and the coverage ratio.
Args:
cv_text: the candidate's CV text.
posting_text: the job posting text.
Returns:
Dict with keys:
- ``matched``: list of posting keywords found in CV.
- ``missing``: list of posting keywords NOT found in CV.
- ``ratio``: float (matched / total), 0.0 if no keywords.
"""
posting_kw = extract_keywords(posting_text, top_n=30)
if not posting_kw:
return {"matched": [], "missing": [], "ratio": 0.0}
cv_kw_set: set[str] = set(extract_keywords(cv_text, top_n=200))
matched: list[str] = []
missing: list[str] = []
for kw in posting_kw:
if kw in cv_kw_set:
matched.append(kw)
else:
missing.append(kw)
total = len(posting_kw)
ratio = len(matched) / total if total > 0 else 0.0
return {
"matched": matched,
"missing": missing,
"ratio": ratio,
}

View file

@ -1,113 +0,0 @@
"""Similarity helpers for job postings.
Uses rapidfuzz for fuzzy string matching. All functions are deterministic.
"""
from __future__ import annotations
import re
from rapidfuzz import fuzz
# ---------------------------------------------------------------------------
# Agency-suffix / noise words to strip from employer names.
# Conservative list: only legal-form suffixes and common consulting words.
# ---------------------------------------------------------------------------
_AGENCY_SUFFIXES: list[str] = [
# Swedish legal forms
"ab",
"aktiebolag",
"hb",
"kb",
"ekonomisk forening",
# English legal forms
"inc",
"corp",
"corporation",
"ltd",
"limited",
"llc",
"gmbh",
"ag",
"sas",
"sarl",
# Consulting / staffing suffixes (agency hints)
"consulting",
"consultancy",
"consult",
"recruitment",
"staffing",
"solutions",
"services",
"group",
"partners",
]
# Pre-compile regex for trailing suffix removal.
_SUFFIX_RE = re.compile(
r"\s+(" + "|".join(re.escape(s) for s in _AGENCY_SUFFIXES) + r")\.?\s*$",
flags=re.IGNORECASE,
)
# Characters to collapse: punctuation -> space, then multi-space -> single.
_PUNCT_RE = re.compile(r"[^\w\s]")
_WS_RE = re.compile(r"\s+")
def normalize_employer(name: str) -> str:
"""Normalize an employer/company name for comparison.
Steps:
1. lowercase
2. strip trailing agency/legal suffixes (ab, consulting, etc.)
3. remove punctuation
4. collapse whitespace
Examples:
>>> normalize_employer("Acme Consulting AB")
'acme'
>>> normalize_employer("Acme AB")
'acme'
>>> normalize_employer(" Globex Corp. ")
'globex'
"""
if not name:
return ""
s = name.strip().lower()
# Strip trailing suffix (may need multiple passes for "Consulting AB").
for _ in range(3):
new = _SUFFIX_RE.sub("", s)
if new == s:
break
s = new
# Remove punctuation.
s = _PUNCT_RE.sub(" ", s)
s = _WS_RE.sub(" ", s).strip()
return s
def title_score(a: str, b: str) -> float:
"""Token-set ratio score for two job titles (0-100).
Uses rapidfuzz ``fuzz.token_set_ratio`` which is order-independent
and handles subsets well.
"""
if not a or not b:
return 0.0
return float(fuzz.token_set_ratio(a, b))
def employer_match(a: str, b: str) -> bool:
"""True if two employer names normalize to the same string."""
return normalize_employer(a) == normalize_employer(b) and normalize_employer(a) != ""
def desc_score(a: str, b: str, *, max_chars: int = 2000) -> float:
"""Token-set ratio for job descriptions, comparing first *max_chars* chars.
Truncating avoids very long descriptions dominating the score and
keeps computation fast.
"""
if not a or not b:
return 0.0
return float(fuzz.token_set_ratio(a[:max_chars], b[:max_chars]))

View file

@ -1,184 +0,0 @@
"""Shared fixtures for matching tests.
Provides agency-repost fixture triples and a legit-different-jobs-same-agency
negative case.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# Agency-repost fixture triples.
# Three realistic scenarios where agencies repost the same job.
# Each triple is a list of 3 posting dicts that should all cluster together.
# ---------------------------------------------------------------------------
# Triple 1: Same role, two agencies + direct employer posting.
# All three have the same title and nearly identical description, but
# different employer names (the two agencies vs the actual company).
TRIPLE_1_AGENCY_REPOST = [
{
"id": "t1-a",
"employer": "TechCorp AB",
"title": "Senior Python Developer",
"description": (
"We are looking for a Senior Python Developer to join our backend team. "
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
"environment. 5+ years of Python experience required. "
"Experience with AWS and Kubernetes is a plus."
),
},
{
"id": "t1-b",
"employer": "Nordic IT Consulting AB",
"title": "Senior Python Developer",
"description": (
"We are looking for a Senior Python Developer to join our backend team. "
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
"environment. 5+ years of Python experience required. "
"Experience with AWS and Kubernetes is a plus."
),
},
{
"id": "t1-c",
"employer": "Acme Recruitment Group",
"title": "Senior Python Developer",
"description": (
"We are looking for a Senior Python Developer to join our backend team. "
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
"environment. 5+ years of Python experience required. "
"Experience with AWS and Kubernetes is a plus."
),
},
]
# Triple 2: Same role reposted by same agency with minor wording variations.
TRIPLE_2_SAME_AGENCY_REPOST = [
{
"id": "t2-a",
"employer": "Stockholm Tech Staffing AB",
"title": "Fullstack Engineer",
"description": (
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
"You will build customer-facing features and internal tools. "
"Must have experience with CI/CD pipelines."
),
},
{
"id": "t2-b",
"employer": "Stockholm Tech Staffing AB",
"title": "Fullstack Engineer",
"description": (
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
"You will build customer-facing features and internal tools. "
"Must have experience with CI/CD pipelines."
),
},
{
"id": "t2-c",
"employer": "Stockholm Tech Staffing",
"title": "Fullstack Engineer",
"description": (
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
"You will build customer-facing features and internal tools. "
"Must have experience with CI/CD pipelines."
),
},
]
# Triple 3: Same role, slightly different title but same description body.
# Different employers (agencies), high title + desc similarity.
TRIPLE_3_CROSS_AGENCY = [
{
"id": "t3-a",
"employer": "Data Recruiting Solutions",
"title": "Data Engineer",
"description": (
"We seek a Data Engineer to build and maintain ETL pipelines using "
"Python, Airflow, dbt, and Snowflake. You will design data models, "
"optimize queries, and ensure data quality. Experience with "
"distributed systems and Spark is required."
),
},
{
"id": "t3-b",
"employer": "Cloud Talent Partners",
"title": "Data Engineer",
"description": (
"We seek a Data Engineer to build and maintain ETL pipelines using "
"Python, Airflow, dbt, and Snowflake. You will design data models, "
"optimize queries, and ensure data quality. Experience with "
"distributed systems and Spark is required."
),
},
{
"id": "t3-c",
"employer": "Analytics Staffing Ltd",
"title": "Data Engineer",
"description": (
"We seek a Data Engineer to build and maintain ETL pipelines using "
"Python, Airflow, dbt, and Snowflake. You will design data models, "
"optimize queries, and ensure data quality. Experience with "
"distributed systems and Spark is required."
),
},
]
# ---------------------------------------------------------------------------
# NEGATIVE case: legit different jobs at same agency -- must NOT cluster.
# Same agency employer but different titles and different descriptions.
# ---------------------------------------------------------------------------
NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY = [
{
"id": "neg-a",
"employer": "Nordic IT Consulting AB",
"title": "Frontend Developer",
"description": (
"We are looking for a Frontend Developer with expertise in React "
"and TypeScript. You will build responsive web applications and "
"work closely with our design team. Experience with CSS-in-JS and "
"accessibility standards is required."
),
},
{
"id": "neg-b",
"employer": "Nordic IT Consulting AB",
"title": "DevOps Engineer",
"description": (
"We need a DevOps Engineer to manage our Kubernetes clusters and "
"CI/CD pipelines. You will work with Terraform, ArgoCD, and "
"Prometheus monitoring. Strong Linux and networking background "
"is required. AWS certification is a plus."
),
},
]
@pytest.fixture
def triple1():
"""Agency repost triple 1: same role, two agencies + employer."""
return [dict(p) for p in TRIPLE_1_AGENCY_REPOST]
@pytest.fixture
def triple2():
"""Agency repost triple 2: same agency reposts same job."""
return [dict(p) for p in TRIPLE_2_SAME_AGENCY_REPOST]
@pytest.fixture
def triple3():
"""Agency repost triple 3: cross-agency same role same description."""
return [dict(p) for p in TRIPLE_3_CROSS_AGENCY]
@pytest.fixture
def negative_same_agency():
"""Negative case: different jobs at same agency, must NOT cluster."""
return [dict(p) for p in NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY]

View file

@ -1,168 +0,0 @@
"""Tests for dedupe clustering."""
from __future__ import annotations
from matching.dedupe import cluster
class TestClusterBasic:
def test_empty_list(self):
assert cluster([]) == {}
def test_single_posting(self):
result = cluster([
{"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}
])
assert len(result) == 1
assert "c1" in result
assert result["c1"] == ["a"]
def test_no_duplicates_separate_clusters(self):
postings = [
{"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"},
{"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"},
{"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"},
]
result = cluster(postings)
# Each posting in its own cluster.
total_ids = sum(len(v) for v in result.values())
assert total_ids == 3
# All cluster values are singletons.
for ids in result.values():
assert len(ids) == 1
class TestClusterEmployerMatch:
def test_same_employer_clusters(self):
"""Same employer name (normalized) with similar content should cluster."""
postings = [
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
{"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
]
result = cluster(postings)
# Same employer + similar title -> same cluster.
assert len(result) == 1
assert set(result["c1"]) == {"a", "b"}
def test_same_employer_different_content_no_cluster(self):
"""Same employer but completely different titles/descriptions should NOT cluster."""
postings = [
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."},
{"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."},
]
result = cluster(postings)
# Same employer but different jobs -> no cluster.
assert all(len(v) == 1 for v in result.values())
def test_employer_suffix_variations_cluster(self):
"""Acme AB and Acme should cluster (normalized match)."""
postings = [
{"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"},
{"id": "b", "employer": "Acme", "title": "Dev", "description": "y"},
]
result = cluster(postings)
assert len(result) == 1
assert set(result["c1"]) == {"a", "b"}
class TestClusterTitleDescMatch:
def test_title_desc_high_enough(self):
"""Different employers but title >= 85 and desc >= 80 -> cluster."""
desc = (
"We are looking for a Senior Python Developer to join our backend "
"team. You will work with Fast API, PostgreSQL, and Docker."
)
postings = [
{"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc},
{"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc},
]
result = cluster(postings)
assert len(result) == 1
assert set(result["c1"]) == {"a", "b"}
def test_title_high_desc_low_no_cluster(self):
"""Title similar but desc too different -> no cluster."""
postings = [
{"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."},
{"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."},
]
result = cluster(postings)
# Should NOT cluster (different employers, low desc score).
assert len(result) == 2 or all(len(v) == 1 for v in result.values())
class TestClusterTransitive:
def test_transitive_clustering(self):
"""If A~B and B~C then A~C should be in same cluster."""
# A and B same employer + similar title, B and C same employer + similar title.
postings = [
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
{"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
{"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"},
]
result = cluster(postings)
assert len(result) == 1
assert set(result["c1"]) == {"a", "b", "c"}
class TestClusterSorting:
def test_cluster_ids_sorted_by_score(self):
"""Cluster with higher pairwise score should get c1."""
# Tight cluster: identical titles and descriptions (different employers).
tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes."
# Looser cluster: different employers, high title but lower desc similarity.
loose_desc_1 = "Python data engineering and pipelines with ETL tools."
loose_desc_2 = "Python data engineering and ETL work with Airflow."
postings = [
# tight cluster (different employers, high title+desc)
{"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc},
{"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc},
# loose cluster (different employers, high title but lower desc)
{"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1},
{"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2},
]
result = cluster(postings)
# Both clusters should exist.
all_ids = set()
for ids in result.values():
all_ids.update(ids)
assert all_ids == {"t1", "t2", "l1", "l2"}
# c1 should be the tight cluster (higher score: title+desc both 100).
assert set(result["c1"]) == {"t1", "t2"}
class TestAgencyRepostTriples:
"""Test the 3 agency-repost fixture triples."""
def test_triple1_all_cluster(self, triple1):
"""Triple 1: 3 postings of same role via different employers cluster."""
result = cluster(triple1)
assert len(result) == 1
assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"}
def test_triple2_all_cluster(self, triple2):
"""Triple 2: same agency reposts same job (suffix variations)."""
result = cluster(triple2)
assert len(result) == 1
assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"}
def test_triple3_all_cluster(self, triple3):
"""Triple 3: cross-agency same role same description."""
result = cluster(triple3)
assert len(result) == 1
assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"}
class TestNegativeSameAgencyDifferentJobs:
"""Negative case: different jobs at same agency must NOT cluster."""
def test_different_jobs_same_agency_no_cluster(self, negative_same_agency):
"""Different jobs at same agency must NOT cluster.
Same employer but completely different titles and descriptions.
Per the clustering rule, same employer alone is not sufficient;
some content overlap (title >= 85 OR desc >= 80) is also required.
"""
result = cluster(negative_same_agency)
all_singletons = all(len(v) == 1 for v in result.values())
assert all_singletons, "Different jobs at same agency should not cluster"

View file

@ -1,152 +0,0 @@
"""Tests for keyword extraction and coverage."""
from __future__ import annotations
from matching.keywords import extract_keywords, coverage
class TestExtractKeywords:
def test_basic_extraction(self):
text = "Python developer with Fast API experience and PostgreSQL database skills."
kws = extract_keywords(text)
assert "python" in kws
assert "fastapi" in kws
assert "postgresql" in kws
def test_stopwords_removed(self):
text = "We are looking for a developer with experience in Python."
kws = extract_keywords(text)
assert "we" not in kws
assert "are" not in kws
assert "for" not in kws
assert "a" not in kws
assert "in" not in kws
assert "python" in kws
assert "developer" in kws
def test_swedish_stopwords_removed(self):
text = "Vi letar efter en Python utvecklare med erfarenhet av Docker."
kws = extract_keywords(text)
assert "vi" not in kws
assert "en" not in kws
assert "av" not in kws
assert "python" in kws
assert "docker" in kws
def test_top_n_limit(self):
text = "python python python docker docker docker kubernetes kubernetes kubernetes react react react"
kws = extract_keywords(text, top_n=2)
assert len(kws) == 2
def test_empty_text(self):
assert extract_keywords("") == []
def test_whitespace_only(self):
assert extract_keywords(" ") == []
def test_multiword_fastapi(self):
text = "Experience with fast api framework for building REST APIs."
kws = extract_keywords(text)
assert "fastapi" in kws
def test_multiword_machine_learning(self):
text = "machine learning models for predictive analytics."
kws = extract_keywords(text)
assert "machine-learning" in kws
def test_frequency_ordering(self):
text = "python python python docker docker kubernetes"
kws = extract_keywords(text, top_n=3)
assert kws[0] == "python"
assert kws[1] == "docker"
assert kws[2] == "kubernetes"
def test_deterministic_tie_breaking(self):
"""Ties in frequency should be broken alphabetically."""
text = "docker kubernetes"
kws = extract_keywords(text)
# Both have frequency 1, so alphabetical: docker < kubernetes
assert kws[0] == "docker"
assert kws[1] == "kubernetes"
def test_min_token_length(self):
text = "x y z aa bb cc developer"
kws = extract_keywords(text)
assert "x" not in kws
assert "y" not in kws
assert "z" not in kws
assert "developer" in kws
def test_tech_terms_preserved(self):
text = "Node.js and React Native for mobile development."
kws = extract_keywords(text)
assert "nodejs" in kws
assert "react-native" in kws
class TestCoverage:
def test_full_coverage(self):
cv = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
result = coverage(cv, posting)
assert result["ratio"] == 1.0
assert len(result["missing"]) == 0
def test_partial_coverage(self):
cv = "Python developer with PostgreSQL and Docker experience"
posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes React"
result = coverage(cv, posting)
assert 0.0 < result["ratio"] < 1.0
assert "python" in result["matched"]
assert "postgresql" in result["matched"]
assert "docker" in result["matched"]
assert "fastapi" in result["missing"]
assert "kubernetes" in result["missing"]
def test_zero_coverage(self):
cv = "Chef with experience in French cuisine and menu planning"
posting = "Python developer with Fast API PostgreSQL Docker"
result = coverage(cv, posting)
assert result["ratio"] == 0.0
assert len(result["matched"]) == 0
assert len(result["missing"]) > 0
def test_empty_posting(self):
result = coverage("Python developer", "")
assert result == {"matched": [], "missing": [], "ratio": 0.0}
def test_empty_cv(self):
result = coverage("", "Python developer with Docker")
assert result["ratio"] == 0.0
assert len(result["matched"]) == 0
assert len(result["missing"]) > 0
def test_both_empty(self):
result = coverage("", "")
assert result == {"matched": [], "missing": [], "ratio": 0.0}
def test_ratio_calculation(self):
cv = "python docker postgresql"
posting = "python docker postgresql kubernetes"
result = coverage(cv, posting)
# 3 of 4 matched (approx, depends on stopword filtering).
assert result["ratio"] > 0.5
assert result["ratio"] <= 1.0
def test_matched_and_missing_lists(self):
cv = "python docker"
posting = "python docker kubernetes react"
result = coverage(cv, posting)
assert "python" in result["matched"]
assert "docker" in result["matched"]
assert "kubernetes" in result["missing"]
assert "react" in result["missing"]
def test_coverage_returns_dict_keys(self):
result = coverage("python", "python docker")
assert "matched" in result
assert "missing" in result
assert "ratio" in result
assert isinstance(result["matched"], list)
assert isinstance(result["missing"], list)
assert isinstance(result["ratio"], float)

View file

@ -1,126 +0,0 @@
"""Tests for similarity helpers."""
from __future__ import annotations
from matching.similarity import (
normalize_employer,
title_score,
employer_match,
desc_score,
)
class TestNormalizeEmployer:
def test_simple_lowercase(self):
assert normalize_employer("Acme") == "acme"
def test_strips_swedish_ab(self):
assert normalize_employer("Acme AB") == "acme"
def test_strips_aktiebolag(self):
assert normalize_employer("Acme Aktiebolag") == "acme"
def test_strips_consulting_suffix(self):
assert normalize_employer("Nordic IT Consulting AB") == "nordic it"
def test_strips_corp_suffix(self):
assert normalize_employer("Globex Corp.") == "globex"
def test_strips_ltd_suffix(self):
assert normalize_employer("Foo Ltd") == "foo"
def test_strips_recruitment_suffix(self):
assert normalize_employer("Acme Recruitment Group") == "acme"
def test_strips_multiple_suffixes(self):
# "Consulting AB" should strip both "AB" then "Consulting"
assert normalize_employer("Nordic Consulting AB") == "nordic"
def test_removes_punctuation(self):
assert normalize_employer("Acme, Inc.") == "acme"
def test_empty_string(self):
assert normalize_employer("") == ""
def test_whitespace_only(self):
assert normalize_employer(" ") == ""
def test_preserves_core_name_with_special_chars(self):
result = normalize_employer("Café Nu AB")
assert "café" in result or "cafe" in result
def test_dots_in_name_preserved(self):
# Punctuation (except & which gets stripped) is removed; H&M -> h m
result = normalize_employer("H&M AB")
assert result == "h m"
class TestTitleScore:
def test_identical_titles(self):
assert title_score("Senior Python Developer", "Senior Python Developer") == 100.0
def test_similar_titles_high_score(self):
score = title_score("Python Developer", "Senior Python Developer")
assert score >= 85.0
def test_different_titles_low_score(self):
score = title_score("Python Developer", "Frontend Designer")
assert score < 50.0
def test_empty_title(self):
assert title_score("", "Something") == 0.0
def test_both_empty(self):
assert title_score("", "") == 0.0
def test_order_independent(self):
# token_set_ratio is order-independent
a = "Senior Python Developer"
b = "Developer Python Senior"
assert title_score(a, b) == 100.0
class TestEmployerMatch:
def test_same_name_matches(self):
assert employer_match("Acme AB", "Acme AB") is True
def test_suffix_variation_matches(self):
assert employer_match("Acme AB", "Acme") is True
def test_different_employers_no_match(self):
assert employer_match("Acme AB", "Globex AB") is False
def test_consulting_variations_match(self):
assert employer_match("Nordic IT Consulting AB", "Nordic IT") is True
def test_empty_no_match(self):
assert employer_match("", "") is False
def test_one_empty_no_match(self):
assert employer_match("Acme", "") is False
class TestDescScore:
def test_identical_descriptions(self):
desc = "We are looking for a Python developer with 5 years experience."
assert desc_score(desc, desc) == 100.0
def test_similar_descriptions_high(self):
a = "We are looking for a Python developer with 5 years experience."
b = "We are looking for a Python developer with 5 years experience in web."
assert desc_score(a, b) >= 80.0
def test_different_descriptions_low(self):
a = "We need a frontend developer skilled in React and CSS."
b = "Looking for a data scientist with Python and SQL expertise."
assert desc_score(a, b) < 50.0
def test_empty_desc(self):
assert desc_score("", "something") == 0.0
def test_truncation(self):
# Test that truncation to max_chars works.
long_a = "Python " * 1000
long_b = "Python " * 1000
score = desc_score(long_a, long_b, max_chars=100)
assert score == 100.0