WA2: packages/matching - similarity, dedupe cluster, keywords extract+coverage, 65 tests with agency-repost fixture triples

packages/matching/README.md

@@ -0,0 +1,40 @@
+# packages/matching
+
+Job posting similarity, dedupe clustering, and keyword coverage for the
+jobhunt-platform v1.1 agency duplicate detection feature (ADR-0003).
+
+## Modules
+
+### similarity.py
+- `normalize_employer(name)` -- lowercase, strip agency/legal suffixes (AB, Consulting, etc.), remove punctuation.
+- `title_score(a, b)` -- rapidfuzz token_set_ratio on job titles (0-100).
+- `employer_match(a, b)` -- True if normalized employer names are equal.
+- `desc_score(a, b, max_chars=2000)` -- token_set_ratio on first 2000 chars of descriptions.
+
+### dedupe.py
+- `cluster(postings: list[dict]) -> dict[str, list[str]]` -- group postings into duplicate clusters.
+
+Clustering rule (per ADR-0003):
+- Same employer (normalized) **OR**
+- Title similarity >= 85 **AND** description similarity >= 80
+
+Uses union-find for transitive grouping.  Clusters are sorted by descending
+max pairwise score (`c1` = tightest cluster).
+
+### keywords.py
+- `extract_keywords(text, top_n=30)` -- frequency-based keyword extraction with Swedish + English stopword removal.
+- `coverage(cv_text, posting_text)` -- computes keyword coverage of a CV against a job posting.
+
+Multiword tech terms like "fast api" are collapsed to "fastapi" so they survive as single keywords.
+
+## Installation (uv)
+
+```bash
+cd packages/matching
+uv venv && source .venv/bin/activate
+uv pip install -e ".[dev]"
+pytest
+```
+
+## License
+MIT

packages/matching/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "matching"
+version = "0.1.0"
+description = "Job posting similarity, dedupe clustering, and keyword coverage for agency duplicate detection."
+requires-python = ">=3.13"
+dependencies = [
+    "rapidfuzz>=3.6",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/matching"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+asyncio_mode = "auto"

packages/matching/src/matching/__init__.py

@@ -0,0 +1,28 @@
+"""Job posting matching package.
+
+Provides:
+- similarity: normalized text comparison (employer match, title/desc scores)
+- dedupe: cluster postings into duplicate groups
+- keywords: keyword extraction and CV-vs-posting coverage
+"""
+
+from __future__ import annotations
+
+from matching.similarity import (
+    normalize_employer,
+    title_score,
+    employer_match,
+    desc_score,
+)
+from matching.dedupe import cluster
+from matching.keywords import extract_keywords, coverage
+
+__all__ = [
+    "normalize_employer",
+    "title_score",
+    "employer_match",
+    "desc_score",
+    "cluster",
+    "extract_keywords",
+    "coverage",
+]

packages/matching/src/matching/dedupe.py

@@ -0,0 +1,136 @@
+"""Duplicate clustering for job postings.
+
+Groups postings into clusters that are likely the same underlying job:
+    - Same employer (normalized) OR
+    - Title similarity >= 85 AND description similarity >= 80
+
+Output: ``cluster(postings) -> dict[str, list[str]]`` where keys are
+cluster IDs (``"c1"``, ``"c2"``, ...) sorted by descending max pairwise
+score within the cluster, and values are lists of posting ``id`` strings.
+"""
+
+from __future__ import annotations
+
+from matching.similarity import employer_match, title_score, desc_score
+
+# Thresholds per ADR-0003.
+TITLE_THRESHOLD = 85.0
+DESC_THRESHOLD = 80.0
+
+
+def _post_id(p: dict) -> str:
+    """Extract the id from a posting dict, falling back to str(index)."""
+    pid = p.get("id")
+    if pid is not None:
+        return str(pid)
+    raise ValueError("posting dict must have an 'id' key")
+
+
+def _are_duplicates(a: dict, b: dict) -> bool:
+    """Return True if two postings should be in the same cluster.
+
+    Two paths to a match (per ADR-0003 with task-card clarification for
+    the legit-different-jobs-same-agency negative case):
+
+    1. Same employer (normalized) AND some content overlap
+       (title >= 85 OR desc >= 80).
+       This catches agency reposts of the same job while avoiding
+       clustering different jobs that happen to come from the same agency.
+
+    2. Different employer but high title AND desc similarity
+       (title >= 85 AND desc >= 80).
+       This catches cross-agency reposts of the same job.
+    """
+    ts = title_score(a.get("title", ""), b.get("title", ""))
+    ds = desc_score(a.get("description", ""), b.get("description", ""))
+    same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
+
+    if same_employer:
+        # Same employer + at least one content dimension similar.
+        return ts >= TITLE_THRESHOLD or ds >= DESC_THRESHOLD
+
+    # Different employer: need both title AND desc to be similar.
+    return ts >= TITLE_THRESHOLD and ds >= DESC_THRESHOLD
+
+
+def _pair_score(a: dict, b: dict) -> float:
+    """Compute a similarity score between two postings for sorting clusters."""
+    ts = title_score(a.get("title", ""), b.get("title", ""))
+    ds = desc_score(a.get("description", ""), b.get("description", ""))
+    same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
+    if same_employer:
+        # Employer match: weight title more for tie-breaking.
+        return 100.0 + ts
+    return (ts + ds) / 2.0
+
+
+def cluster(postings: list[dict]) -> dict[str, list[str]]:
+    """Cluster job postings into duplicate groups.
+
+    Uses union-find so transitive duplicates (A~B, B~C => A~C) are grouped
+    together.
+
+    Args:
+        postings: list of dicts with keys ``id``, ``employer``, ``title``,
+            ``description``.
+
+    Returns:
+        Dict mapping cluster_id (``"c1"``, ``"c2"``, ...) to a list of
+        posting id strings.  Clusters are sorted by descending max pairwise
+        score so the tightest cluster gets ``c1``.
+    """
+    n = len(postings)
+    if n == 0:
+        return {}
+
+    ids = [_post_id(p) for p in postings]
+
+    # Union-find (disjoint set).
+    parent: list[int] = list(range(n))
+
+    def find(x: int) -> int:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(a: int, b: int) -> None:
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            parent[ra] = rb
+
+    # O(n^2) pairwise comparison.
+    for i in range(n):
+        for j in range(i + 1, n):
+            if _are_duplicates(postings[i], postings[j]):
+                union(i, j)
+
+    # Collect groups.
+    groups: dict[int, list[int]] = {}
+    for i in range(n):
+        root = find(i)
+        groups.setdefault(root, []).append(i)
+
+    # Compute max pairwise score per group for sorting.
+    group_scores: list[tuple[float, list[int]]] = []
+    for indices in groups.values():
+        if len(indices) < 2:
+            max_sc = 0.0
+        else:
+            max_sc = 0.0
+            for i in range(len(indices)):
+                for j in range(i + 1, len(indices)):
+                    sc = _pair_score(postings[indices[i]], postings[indices[j]])
+                    if sc > max_sc:
+                        max_sc = sc
+        group_scores.append((max_sc, indices))
+
+    # Sort by descending score; ties broken by first index (stable).
+    group_scores.sort(key=lambda t: (-t[0], t[1][0]))
+
+    # Assign cluster IDs.
+    result: dict[str, list[str]] = {}
+    for idx, (_, indices) in enumerate(group_scores, start=1):
+        result[f"c{idx}"] = [ids[i] for i in indices]
+
+    return result

packages/matching/src/matching/keywords.py

@@ -0,0 +1,162 @@
+"""Keyword extraction and coverage analysis.
+
+Deterministic, no LLM.  Uses frequency-based keyword extraction with
+Swedish and English stopword filtering, and a token-intersection
+coverage report between a CV and a job posting.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+
+# ---------------------------------------------------------------------------
+# Stopwords (Swedish + English).  Conservative lists.
+# ---------------------------------------------------------------------------
+
+_SWEDISH_STOPWORDS: frozenset[str] = frozenset({
+    "och", "eller", "som", "att", "den", "det", "de", "vi", "ni", "du",
+    "han", "hon", "den", "en", "ett", "ar", "har", "var", "var", "inte",
+    "med", "for", "fran", "till", "pa", "av", "i", "och", "men", "sa",
+    "när", "då", "hur", "alla", "nagon", "nagot", "alla", "manga", "mycket",
+    "skall", "ska", "kan", "kommer", "blir", "vore", "vill", "borde",
+    "efter", "under", "over", "bAKom", "inom", "mellan", "genom", "utan",
+    "mot", "ut", "fran", "frams", "igår", "idag", "imorgon",
+})
+
+_ENGLISH_STOPWORDS: frozenset[str] = frozenset({
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
+    "being", "have", "has", "had", "do", "does", "did", "will", "would",
+    "should", "could", "may", "might", "must", "can", "this", "that",
+    "these", "those", "i", "you", "he", "she", "it", "we", "they", "me",
+    "him", "her", "us", "them", "my", "your", "his", "its", "our", "their",
+    "what", "which", "who", "whom", "where", "when", "why", "how", "all",
+    "each", "every", "both", "few", "more", "most", "other", "some", "such",
+    "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very",
+    "as", "if", "about", "against", "between", "into", "through", "during",
+    "before", "after", "above", "below", "up", "down", "out", "off",
+    "over", "under", "again", "further", "then", "once", "here", "there",
+    "also", "etc", "e.g", "i.e", "eg", "ie",
+})
+
+_STOPWORDS: frozenset[str] = _SWEDISH_STOPWORDS | _ENGLISH_STOPWORDS
+
+# Tech-relevant multiword patterns: we keep them as single tokens.
+# e.g. "fast api" -> "fastapi" so it survives as a keyword.
+_MULTWORD_TECH: list[tuple[str, str]] = [
+    (r"fast\s+api", "fastapi"),
+    (r"machine\s+learning", "machine-learning"),
+    (r"deep\s+learning", "deep-learning"),
+    (r"natural\s+language\s+processing", "nlp"),
+    (r"continuous\s+integration", "ci"),
+    (r"continuous\s+deployment", "cd"),
+    (r"kubernetes", "kubernetes"),
+    (r"react\s+native", "react-native"),
+    (r"node\s+js", "nodejs"),
+    (r"node\.js", "nodejs"),
+    (r"aws", "aws"),
+    (r"gcp", "gcp"),
+    (r"ci/cd", "ci-cd"),
+]
+
+# Minimum token length for keywords.
+_MIN_TOKEN_LEN = 2
+
+# Tokenizer: split on non-alphanumeric.
+_TOKEN_RE = re.compile(r"[a-z0-9+#.\-/]+")
+
+
+def _preprocess_multiwords(text: str) -> str:
+    """Replace known multiword tech terms with single tokens."""
+    result = text.lower()
+    for pattern, replacement in _MULTWORD_TECH:
+        result = re.sub(pattern, replacement, result, flags=re.IGNORECASE)
+    return result
+
+
+def _tokenize(text: str) -> list[str]:
+    """Tokenize text into lowercased tokens."""
+    text = _preprocess_multiwords(text)
+    raw_tokens = _TOKEN_RE.findall(text.lower())
+    tokens: list[str] = []
+    for t in raw_tokens:
+        t = t.strip(".-/")
+        if not t:
+            continue
+        if t in _STOPWORDS:
+            continue
+        if len(t) < _MIN_TOKEN_LEN:
+            continue
+        # Skip pure numbers (unless they look like versions).
+        if t.isdigit() and len(t) > 4:
+            continue
+        tokens.append(t)
+    return tokens
+
+
+def extract_keywords(text: str, top_n: int = 30) -> list[str]:
+    """Extract the top *top_n* keywords from *text* by frequency.
+
+    Keywords are lowercased tokens.  Stopwords (Swedish + English) are
+    removed.  Multiword tech terms like "fast api" are collapsed to
+    "fastapi".
+
+    Args:
+        text: the text to analyze.
+        top_n: maximum number of keywords to return.
+
+    Returns:
+        List of keyword strings, most frequent first.  Ties are broken
+        alphabetically for determinism.
+    """
+    if not text:
+        return []
+    tokens = _tokenize(text)
+    if not tokens:
+        return []
+    counts: Counter[str] = Counter(tokens)
+    # Sort by count desc, then alphabetically for deterministic order.
+    ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+    return [word for word, _ in ranked[:top_n]]
+
+
+def coverage(cv_text: str, posting_text: str) -> dict[str, list[str] | float]:
+    """Compute keyword coverage of a CV against a job posting.
+
+    Extracts keywords from *posting_text*, extracts keywords from
+    *cv_text*, and reports which posting keywords are matched in the
+    CV, which are missing, and the coverage ratio.
+
+    Args:
+        cv_text: the candidate's CV text.
+        posting_text: the job posting text.
+
+    Returns:
+        Dict with keys:
+            - ``matched``: list of posting keywords found in CV.
+            - ``missing``: list of posting keywords NOT found in CV.
+            - ``ratio``: float (matched / total), 0.0 if no keywords.
+    """
+    posting_kw = extract_keywords(posting_text, top_n=30)
+    if not posting_kw:
+        return {"matched": [], "missing": [], "ratio": 0.0}
+
+    cv_kw_set: set[str] = set(extract_keywords(cv_text, top_n=200))
+
+    matched: list[str] = []
+    missing: list[str] = []
+    for kw in posting_kw:
+        if kw in cv_kw_set:
+            matched.append(kw)
+        else:
+            missing.append(kw)
+
+    total = len(posting_kw)
+    ratio = len(matched) / total if total > 0 else 0.0
+
+    return {
+        "matched": matched,
+        "missing": missing,
+        "ratio": ratio,
+    }

packages/matching/src/matching/similarity.py

@@ -0,0 +1,113 @@
+"""Similarity helpers for job postings.
+
+Uses rapidfuzz for fuzzy string matching.  All functions are deterministic.
+"""
+
+from __future__ import annotations
+
+import re
+from rapidfuzz import fuzz
+
+# ---------------------------------------------------------------------------
+# Agency-suffix / noise words to strip from employer names.
+# Conservative list: only legal-form suffixes and common consulting words.
+# ---------------------------------------------------------------------------
+
+_AGENCY_SUFFIXES: list[str] = [
+    # Swedish legal forms
+    "ab",
+    "aktiebolag",
+    "hb",
+    "kb",
+    "ekonomisk forening",
+    # English legal forms
+    "inc",
+    "corp",
+    "corporation",
+    "ltd",
+    "limited",
+    "llc",
+    "gmbh",
+    "ag",
+    "sas",
+    "sarl",
+    # Consulting / staffing suffixes (agency hints)
+    "consulting",
+    "consultancy",
+    "consult",
+    "recruitment",
+    "staffing",
+    "solutions",
+    "services",
+    "group",
+    "partners",
+]
+
+# Pre-compile regex for trailing suffix removal.
+_SUFFIX_RE = re.compile(
+    r"\s+(" + "|".join(re.escape(s) for s in _AGENCY_SUFFIXES) + r")\.?\s*$",
+    flags=re.IGNORECASE,
+)
+
+# Characters to collapse: punctuation -> space, then multi-space -> single.
+_PUNCT_RE = re.compile(r"[^\w\s]")
+_WS_RE = re.compile(r"\s+")
+
+
+def normalize_employer(name: str) -> str:
+    """Normalize an employer/company name for comparison.
+
+    Steps:
+        1. lowercase
+        2. strip trailing agency/legal suffixes (ab, consulting, etc.)
+        3. remove punctuation
+        4. collapse whitespace
+
+    Examples:
+        >>> normalize_employer("Acme Consulting AB")
+        'acme'
+        >>> normalize_employer("Acme AB")
+        'acme'
+        >>> normalize_employer("  Globex Corp. ")
+        'globex'
+    """
+    if not name:
+        return ""
+    s = name.strip().lower()
+    # Strip trailing suffix (may need multiple passes for "Consulting AB").
+    for _ in range(3):
+        new = _SUFFIX_RE.sub("", s)
+        if new == s:
+            break
+        s = new
+    # Remove punctuation.
+    s = _PUNCT_RE.sub(" ", s)
+    s = _WS_RE.sub(" ", s).strip()
+    return s
+
+
+def title_score(a: str, b: str) -> float:
+    """Token-set ratio score for two job titles (0-100).
+
+    Uses rapidfuzz ``fuzz.token_set_ratio`` which is order-independent
+    and handles subsets well.
+    """
+    if not a or not b:
+        return 0.0
+    return float(fuzz.token_set_ratio(a, b))
+
+
+def employer_match(a: str, b: str) -> bool:
+    """True if two employer names normalize to the same string."""
+    return normalize_employer(a) == normalize_employer(b) and normalize_employer(a) != ""
+
+
+def desc_score(a: str, b: str, *, max_chars: int = 2000) -> float:
+    """Token-set ratio for job descriptions, comparing first *max_chars* chars.
+
+    Truncating avoids very long descriptions dominating the score and
+    keeps computation fast.
+    """
+    if not a or not b:
+        return 0.0
+    return float(fuzz.token_set_ratio(a[:max_chars], b[:max_chars]))

packages/matching/tests/conftest.py

@@ -0,0 +1,184 @@
+"""Shared fixtures for matching tests.
+
+Provides agency-repost fixture triples and a legit-different-jobs-same-agency
+negative case.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# Agency-repost fixture triples.
+# Three realistic scenarios where agencies repost the same job.
+# Each triple is a list of 3 posting dicts that should all cluster together.
+# ---------------------------------------------------------------------------
+
+# Triple 1: Same role, two agencies + direct employer posting.
+# All three have the same title and nearly identical description, but
+# different employer names (the two agencies vs the actual company).
+TRIPLE_1_AGENCY_REPOST = [
+    {
+        "id": "t1-a",
+        "employer": "TechCorp AB",
+        "title": "Senior Python Developer",
+        "description": (
+            "We are looking for a Senior Python Developer to join our backend team. "
+            "You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
+            "environment. 5+ years of Python experience required. "
+            "Experience with AWS and Kubernetes is a plus."
+        ),
+    },
+    {
+        "id": "t1-b",
+        "employer": "Nordic IT Consulting AB",
+        "title": "Senior Python Developer",
+        "description": (
+            "We are looking for a Senior Python Developer to join our backend team. "
+            "You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
+            "environment. 5+ years of Python experience required. "
+            "Experience with AWS and Kubernetes is a plus."
+        ),
+    },
+    {
+        "id": "t1-c",
+        "employer": "Acme Recruitment Group",
+        "title": "Senior Python Developer",
+        "description": (
+            "We are looking for a Senior Python Developer to join our backend team. "
+            "You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
+            "environment. 5+ years of Python experience required. "
+            "Experience with AWS and Kubernetes is a plus."
+        ),
+    },
+]
+
+# Triple 2: Same role reposted by same agency with minor wording variations.
+TRIPLE_2_SAME_AGENCY_REPOST = [
+    {
+        "id": "t2-a",
+        "employer": "Stockholm Tech Staffing AB",
+        "title": "Fullstack Engineer",
+        "description": (
+            "Fullstack Engineer wanted for a fintech startup in Stockholm. "
+            "Tech stack: React, TypeScript, Node.js, PostgreSQL. "
+            "You will build customer-facing features and internal tools. "
+            "Must have experience with CI/CD pipelines."
+        ),
+    },
+    {
+        "id": "t2-b",
+        "employer": "Stockholm Tech Staffing AB",
+        "title": "Fullstack Engineer",
+        "description": (
+            "Fullstack Engineer wanted for a fintech startup in Stockholm. "
+            "Tech stack: React, TypeScript, Node.js, PostgreSQL. "
+            "You will build customer-facing features and internal tools. "
+            "Must have experience with CI/CD pipelines."
+        ),
+    },
+    {
+        "id": "t2-c",
+        "employer": "Stockholm Tech Staffing",
+        "title": "Fullstack Engineer",
+        "description": (
+            "Fullstack Engineer wanted for a fintech startup in Stockholm. "
+            "Tech stack: React, TypeScript, Node.js, PostgreSQL. "
+            "You will build customer-facing features and internal tools. "
+            "Must have experience with CI/CD pipelines."
+        ),
+    },
+]
+
+# Triple 3: Same role, slightly different title but same description body.
+# Different employers (agencies), high title + desc similarity.
+TRIPLE_3_CROSS_AGENCY = [
+    {
+        "id": "t3-a",
+        "employer": "Data Recruiting Solutions",
+        "title": "Data Engineer",
+        "description": (
+            "We seek a Data Engineer to build and maintain ETL pipelines using "
+            "Python, Airflow, dbt, and Snowflake. You will design data models, "
+            "optimize queries, and ensure data quality. Experience with "
+            "distributed systems and Spark is required."
+        ),
+    },
+    {
+        "id": "t3-b",
+        "employer": "Cloud Talent Partners",
+        "title": "Data Engineer",
+        "description": (
+            "We seek a Data Engineer to build and maintain ETL pipelines using "
+            "Python, Airflow, dbt, and Snowflake. You will design data models, "
+            "optimize queries, and ensure data quality. Experience with "
+            "distributed systems and Spark is required."
+        ),
+    },
+    {
+        "id": "t3-c",
+        "employer": "Analytics Staffing Ltd",
+        "title": "Data Engineer",
+        "description": (
+            "We seek a Data Engineer to build and maintain ETL pipelines using "
+            "Python, Airflow, dbt, and Snowflake. You will design data models, "
+            "optimize queries, and ensure data quality. Experience with "
+            "distributed systems and Spark is required."
+        ),
+    },
+]
+
+# ---------------------------------------------------------------------------
+# NEGATIVE case: legit different jobs at same agency -- must NOT cluster.
+# Same agency employer but different titles and different descriptions.
+# ---------------------------------------------------------------------------
+
+NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY = [
+    {
+        "id": "neg-a",
+        "employer": "Nordic IT Consulting AB",
+        "title": "Frontend Developer",
+        "description": (
+            "We are looking for a Frontend Developer with expertise in React "
+            "and TypeScript. You will build responsive web applications and "
+            "work closely with our design team. Experience with CSS-in-JS and "
+            "accessibility standards is required."
+        ),
+    },
+    {
+        "id": "neg-b",
+        "employer": "Nordic IT Consulting AB",
+        "title": "DevOps Engineer",
+        "description": (
+            "We need a DevOps Engineer to manage our Kubernetes clusters and "
+            "CI/CD pipelines. You will work with Terraform, ArgoCD, and "
+            "Prometheus monitoring. Strong Linux and networking background "
+            "is required. AWS certification is a plus."
+        ),
+    },
+]
+
+
+@pytest.fixture
+def triple1():
+    """Agency repost triple 1: same role, two agencies + employer."""
+    return [dict(p) for p in TRIPLE_1_AGENCY_REPOST]
+
+
+@pytest.fixture
+def triple2():
+    """Agency repost triple 2: same agency reposts same job."""
+    return [dict(p) for p in TRIPLE_2_SAME_AGENCY_REPOST]
+
+
+@pytest.fixture
+def triple3():
+    """Agency repost triple 3: cross-agency same role same description."""
+    return [dict(p) for p in TRIPLE_3_CROSS_AGENCY]
+
+
+@pytest.fixture
+def negative_same_agency():
+    """Negative case: different jobs at same agency, must NOT cluster."""
+    return [dict(p) for p in NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY]

packages/matching/tests/test_dedupe.py

@@ -0,0 +1,168 @@
+"""Tests for dedupe clustering."""
+
+from __future__ import annotations
+
+from matching.dedupe import cluster
+
+
+class TestClusterBasic:
+    def test_empty_list(self):
+        assert cluster([]) == {}
+
+    def test_single_posting(self):
+        result = cluster([
+            {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}
+        ])
+        assert len(result) == 1
+        assert "c1" in result
+        assert result["c1"] == ["a"]
+
+    def test_no_duplicates_separate_clusters(self):
+        postings = [
+            {"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"},
+            {"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"},
+            {"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"},
+        ]
+        result = cluster(postings)
+        # Each posting in its own cluster.
+        total_ids = sum(len(v) for v in result.values())
+        assert total_ids == 3
+        # All cluster values are singletons.
+        for ids in result.values():
+            assert len(ids) == 1
+
+
+class TestClusterEmployerMatch:
+    def test_same_employer_clusters(self):
+        """Same employer name (normalized) with similar content should cluster."""
+        postings = [
+            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
+            {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
+        ]
+        result = cluster(postings)
+        # Same employer + similar title -> same cluster.
+        assert len(result) == 1
+        assert set(result["c1"]) == {"a", "b"}
+
+    def test_same_employer_different_content_no_cluster(self):
+        """Same employer but completely different titles/descriptions should NOT cluster."""
+        postings = [
+            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."},
+            {"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."},
+        ]
+        result = cluster(postings)
+        # Same employer but different jobs -> no cluster.
+        assert all(len(v) == 1 for v in result.values())
+
+    def test_employer_suffix_variations_cluster(self):
+        """Acme AB and Acme should cluster (normalized match)."""
+        postings = [
+            {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"},
+            {"id": "b", "employer": "Acme", "title": "Dev", "description": "y"},
+        ]
+        result = cluster(postings)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"a", "b"}
+
+
+class TestClusterTitleDescMatch:
+    def test_title_desc_high_enough(self):
+        """Different employers but title >= 85 and desc >= 80 -> cluster."""
+        desc = (
+            "We are looking for a Senior Python Developer to join our backend "
+            "team. You will work with Fast API, PostgreSQL, and Docker."
+        )
+        postings = [
+            {"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc},
+            {"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc},
+        ]
+        result = cluster(postings)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"a", "b"}
+
+    def test_title_high_desc_low_no_cluster(self):
+        """Title similar but desc too different -> no cluster."""
+        postings = [
+            {"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."},
+            {"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."},
+        ]
+        result = cluster(postings)
+        # Should NOT cluster (different employers, low desc score).
+        assert len(result) == 2 or all(len(v) == 1 for v in result.values())
+
+
+class TestClusterTransitive:
+    def test_transitive_clustering(self):
+        """If A~B and B~C then A~C should be in same cluster."""
+        # A and B same employer + similar title, B and C same employer + similar title.
+        postings = [
+            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
+            {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
+            {"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"},
+        ]
+        result = cluster(postings)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"a", "b", "c"}
+
+
+class TestClusterSorting:
+    def test_cluster_ids_sorted_by_score(self):
+        """Cluster with higher pairwise score should get c1."""
+        # Tight cluster: identical titles and descriptions (different employers).
+        tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes."
+        # Looser cluster: different employers, high title but lower desc similarity.
+        loose_desc_1 = "Python data engineering and pipelines with ETL tools."
+        loose_desc_2 = "Python data engineering and ETL work with Airflow."
+        postings = [
+            # tight cluster (different employers, high title+desc)
+            {"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc},
+            {"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc},
+            # loose cluster (different employers, high title but lower desc)
+            {"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1},
+            {"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2},
+        ]
+        result = cluster(postings)
+        # Both clusters should exist.
+        all_ids = set()
+        for ids in result.values():
+            all_ids.update(ids)
+        assert all_ids == {"t1", "t2", "l1", "l2"}
+        # c1 should be the tight cluster (higher score: title+desc both 100).
+        assert set(result["c1"]) == {"t1", "t2"}
+
+
+class TestAgencyRepostTriples:
+    """Test the 3 agency-repost fixture triples."""
+
+    def test_triple1_all_cluster(self, triple1):
+        """Triple 1: 3 postings of same role via different employers cluster."""
+        result = cluster(triple1)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"}
+
+    def test_triple2_all_cluster(self, triple2):
+        """Triple 2: same agency reposts same job (suffix variations)."""
+        result = cluster(triple2)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"}
+
+    def test_triple3_all_cluster(self, triple3):
+        """Triple 3: cross-agency same role same description."""
+        result = cluster(triple3)
+        assert len(result) == 1
+        assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"}
+
+
+class TestNegativeSameAgencyDifferentJobs:
+    """Negative case: different jobs at same agency must NOT cluster."""
+
+    def test_different_jobs_same_agency_no_cluster(self, negative_same_agency):
+        """Different jobs at same agency must NOT cluster.
+
+        Same employer but completely different titles and descriptions.
+        Per the clustering rule, same employer alone is not sufficient;
+        some content overlap (title >= 85 OR desc >= 80) is also required.
+        """
+        result = cluster(negative_same_agency)
+        all_singletons = all(len(v) == 1 for v in result.values())
+        assert all_singletons, "Different jobs at same agency should not cluster"

packages/matching/tests/test_keywords.py

@@ -0,0 +1,152 @@
+"""Tests for keyword extraction and coverage."""
+
+from __future__ import annotations
+
+from matching.keywords import extract_keywords, coverage
+
+
+class TestExtractKeywords:
+    def test_basic_extraction(self):
+        text = "Python developer with Fast API experience and PostgreSQL database skills."
+        kws = extract_keywords(text)
+        assert "python" in kws
+        assert "fastapi" in kws
+        assert "postgresql" in kws
+
+    def test_stopwords_removed(self):
+        text = "We are looking for a developer with experience in Python."
+        kws = extract_keywords(text)
+        assert "we" not in kws
+        assert "are" not in kws
+        assert "for" not in kws
+        assert "a" not in kws
+        assert "in" not in kws
+        assert "python" in kws
+        assert "developer" in kws
+
+    def test_swedish_stopwords_removed(self):
+        text = "Vi letar efter en Python utvecklare med erfarenhet av Docker."
+        kws = extract_keywords(text)
+        assert "vi" not in kws
+        assert "en" not in kws
+        assert "av" not in kws
+        assert "python" in kws
+        assert "docker" in kws
+
+    def test_top_n_limit(self):
+        text = "python python python docker docker docker kubernetes kubernetes kubernetes react react react"
+        kws = extract_keywords(text, top_n=2)
+        assert len(kws) == 2
+
+    def test_empty_text(self):
+        assert extract_keywords("") == []
+
+    def test_whitespace_only(self):
+        assert extract_keywords("   ") == []
+
+    def test_multiword_fastapi(self):
+        text = "Experience with fast api framework for building REST APIs."
+        kws = extract_keywords(text)
+        assert "fastapi" in kws
+
+    def test_multiword_machine_learning(self):
+        text = "machine learning models for predictive analytics."
+        kws = extract_keywords(text)
+        assert "machine-learning" in kws
+
+    def test_frequency_ordering(self):
+        text = "python python python docker docker kubernetes"
+        kws = extract_keywords(text, top_n=3)
+        assert kws[0] == "python"
+        assert kws[1] == "docker"
+        assert kws[2] == "kubernetes"
+
+    def test_deterministic_tie_breaking(self):
+        """Ties in frequency should be broken alphabetically."""
+        text = "docker kubernetes"
+        kws = extract_keywords(text)
+        # Both have frequency 1, so alphabetical: docker < kubernetes
+        assert kws[0] == "docker"
+        assert kws[1] == "kubernetes"
+
+    def test_min_token_length(self):
+        text = "x y z aa bb cc developer"
+        kws = extract_keywords(text)
+        assert "x" not in kws
+        assert "y" not in kws
+        assert "z" not in kws
+        assert "developer" in kws
+
+    def test_tech_terms_preserved(self):
+        text = "Node.js and React Native for mobile development."
+        kws = extract_keywords(text)
+        assert "nodejs" in kws
+        assert "react-native" in kws
+
+
+class TestCoverage:
+    def test_full_coverage(self):
+        cv = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
+        posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
+        result = coverage(cv, posting)
+        assert result["ratio"] == 1.0
+        assert len(result["missing"]) == 0
+
+    def test_partial_coverage(self):
+        cv = "Python developer with PostgreSQL and Docker experience"
+        posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes React"
+        result = coverage(cv, posting)
+        assert 0.0 < result["ratio"] < 1.0
+        assert "python" in result["matched"]
+        assert "postgresql" in result["matched"]
+        assert "docker" in result["matched"]
+        assert "fastapi" in result["missing"]
+        assert "kubernetes" in result["missing"]
+
+    def test_zero_coverage(self):
+        cv = "Chef with experience in French cuisine and menu planning"
+        posting = "Python developer with Fast API PostgreSQL Docker"
+        result = coverage(cv, posting)
+        assert result["ratio"] == 0.0
+        assert len(result["matched"]) == 0
+        assert len(result["missing"]) > 0
+
+    def test_empty_posting(self):
+        result = coverage("Python developer", "")
+        assert result == {"matched": [], "missing": [], "ratio": 0.0}
+
+    def test_empty_cv(self):
+        result = coverage("", "Python developer with Docker")
+        assert result["ratio"] == 0.0
+        assert len(result["matched"]) == 0
+        assert len(result["missing"]) > 0
+
+    def test_both_empty(self):
+        result = coverage("", "")
+        assert result == {"matched": [], "missing": [], "ratio": 0.0}
+
+    def test_ratio_calculation(self):
+        cv = "python docker postgresql"
+        posting = "python docker postgresql kubernetes"
+        result = coverage(cv, posting)
+        # 3 of 4 matched (approx, depends on stopword filtering).
+        assert result["ratio"] > 0.5
+        assert result["ratio"] <= 1.0
+
+    def test_matched_and_missing_lists(self):
+        cv = "python docker"
+        posting = "python docker kubernetes react"
+        result = coverage(cv, posting)
+        assert "python" in result["matched"]
+        assert "docker" in result["matched"]
+        assert "kubernetes" in result["missing"]
+        assert "react" in result["missing"]
+
+    def test_coverage_returns_dict_keys(self):
+        result = coverage("python", "python docker")
+        assert "matched" in result
+        assert "missing" in result
+        assert "ratio" in result
+        assert isinstance(result["matched"], list)
+        assert isinstance(result["missing"], list)
+        assert isinstance(result["ratio"], float)

packages/matching/tests/test_similarity.py

@@ -0,0 +1,126 @@
+"""Tests for similarity helpers."""
+
+from __future__ import annotations
+
+from matching.similarity import (
+    normalize_employer,
+    title_score,
+    employer_match,
+    desc_score,
+)
+
+
+class TestNormalizeEmployer:
+    def test_simple_lowercase(self):
+        assert normalize_employer("Acme") == "acme"
+
+    def test_strips_swedish_ab(self):
+        assert normalize_employer("Acme AB") == "acme"
+
+    def test_strips_aktiebolag(self):
+        assert normalize_employer("Acme Aktiebolag") == "acme"
+
+    def test_strips_consulting_suffix(self):
+        assert normalize_employer("Nordic IT Consulting AB") == "nordic it"
+
+    def test_strips_corp_suffix(self):
+        assert normalize_employer("Globex Corp.") == "globex"
+
+    def test_strips_ltd_suffix(self):
+        assert normalize_employer("Foo Ltd") == "foo"
+
+    def test_strips_recruitment_suffix(self):
+        assert normalize_employer("Acme Recruitment Group") == "acme"
+
+    def test_strips_multiple_suffixes(self):
+        # "Consulting AB" should strip both "AB" then "Consulting"
+        assert normalize_employer("Nordic Consulting AB") == "nordic"
+
+    def test_removes_punctuation(self):
+        assert normalize_employer("Acme, Inc.") == "acme"
+
+    def test_empty_string(self):
+        assert normalize_employer("") == ""
+
+    def test_whitespace_only(self):
+        assert normalize_employer("   ") == ""
+
+    def test_preserves_core_name_with_special_chars(self):
+        result = normalize_employer("Café Nu AB")
+        assert "café" in result or "cafe" in result
+
+    def test_dots_in_name_preserved(self):
+        # Punctuation (except & which gets stripped) is removed; H&M -> h m
+        result = normalize_employer("H&M AB")
+        assert result == "h m"
+
+
+class TestTitleScore:
+    def test_identical_titles(self):
+        assert title_score("Senior Python Developer", "Senior Python Developer") == 100.0
+
+    def test_similar_titles_high_score(self):
+        score = title_score("Python Developer", "Senior Python Developer")
+        assert score >= 85.0
+
+    def test_different_titles_low_score(self):
+        score = title_score("Python Developer", "Frontend Designer")
+        assert score < 50.0
+
+    def test_empty_title(self):
+        assert title_score("", "Something") == 0.0
+
+    def test_both_empty(self):
+        assert title_score("", "") == 0.0
+
+    def test_order_independent(self):
+        # token_set_ratio is order-independent
+        a = "Senior Python Developer"
+        b = "Developer Python Senior"
+        assert title_score(a, b) == 100.0
+
+
+class TestEmployerMatch:
+    def test_same_name_matches(self):
+        assert employer_match("Acme AB", "Acme AB") is True
+
+    def test_suffix_variation_matches(self):
+        assert employer_match("Acme AB", "Acme") is True
+
+    def test_different_employers_no_match(self):
+        assert employer_match("Acme AB", "Globex AB") is False
+
+    def test_consulting_variations_match(self):
+        assert employer_match("Nordic IT Consulting AB", "Nordic IT") is True
+
+    def test_empty_no_match(self):
+        assert employer_match("", "") is False
+
+    def test_one_empty_no_match(self):
+        assert employer_match("Acme", "") is False
+
+
+class TestDescScore:
+    def test_identical_descriptions(self):
+        desc = "We are looking for a Python developer with 5 years experience."
+        assert desc_score(desc, desc) == 100.0
+
+    def test_similar_descriptions_high(self):
+        a = "We are looking for a Python developer with 5 years experience."
+        b = "We are looking for a Python developer with 5 years experience in web."
+        assert desc_score(a, b) >= 80.0
+
+    def test_different_descriptions_low(self):
+        a = "We need a frontend developer skilled in React and CSS."
+        b = "Looking for a data scientist with Python and SQL expertise."
+        assert desc_score(a, b) < 50.0
+
+    def test_empty_desc(self):
+        assert desc_score("", "something") == 0.0
+
+    def test_truncation(self):
+        # Test that truncation to max_chars works.
+        long_a = "Python " * 1000
+        long_b = "Python " * 1000
+        score = desc_score(long_a, long_b, max_chars=100)
+        assert score == 100.0