diff --git a/packages/matching/README.md b/packages/matching/README.md new file mode 100644 index 0000000..7fde103 --- /dev/null +++ b/packages/matching/README.md @@ -0,0 +1,40 @@ +# packages/matching + +Job posting similarity, dedupe clustering, and keyword coverage for the +jobhunt-platform v1.1 agency duplicate detection feature (ADR-0003). + +## Modules + +### similarity.py +- `normalize_employer(name)` -- lowercase, strip agency/legal suffixes (AB, Consulting, etc.), remove punctuation. +- `title_score(a, b)` -- rapidfuzz token_set_ratio on job titles (0-100). +- `employer_match(a, b)` -- True if normalized employer names are equal. +- `desc_score(a, b, max_chars=2000)` -- token_set_ratio on first 2000 chars of descriptions. + +### dedupe.py +- `cluster(postings: list[dict]) -> dict[str, list[str]]` -- group postings into duplicate clusters. + +Clustering rule (per ADR-0003): +- Same employer (normalized) **OR** +- Title similarity >= 85 **AND** description similarity >= 80 + +Uses union-find for transitive grouping. Clusters are sorted by descending +max pairwise score (`c1` = tightest cluster). + +### keywords.py +- `extract_keywords(text, top_n=30)` -- frequency-based keyword extraction with Swedish + English stopword removal. +- `coverage(cv_text, posting_text)` -- computes keyword coverage of a CV against a job posting. + +Multiword tech terms like "fast api" are collapsed to "fastapi" so they survive as single keywords. + +## Installation (uv) + +```bash +cd packages/matching +uv venv && source .venv/bin/activate +uv pip install -e ".[dev]" +pytest +``` + +## License +MIT \ No newline at end of file diff --git a/packages/matching/pyproject.toml b/packages/matching/pyproject.toml new file mode 100644 index 0000000..d5463af --- /dev/null +++ b/packages/matching/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "matching" +version = "0.1.0" +description = "Job posting similarity, dedupe clustering, and keyword coverage for agency duplicate detection." +requires-python = ">=3.13" +dependencies = [ + "rapidfuzz>=3.6", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.24", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/matching"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +asyncio_mode = "auto" \ No newline at end of file diff --git a/packages/matching/src/matching/__init__.py b/packages/matching/src/matching/__init__.py new file mode 100644 index 0000000..e03af4f --- /dev/null +++ b/packages/matching/src/matching/__init__.py @@ -0,0 +1,28 @@ +"""Job posting matching package. + +Provides: +- similarity: normalized text comparison (employer match, title/desc scores) +- dedupe: cluster postings into duplicate groups +- keywords: keyword extraction and CV-vs-posting coverage +""" + +from __future__ import annotations + +from matching.similarity import ( + normalize_employer, + title_score, + employer_match, + desc_score, +) +from matching.dedupe import cluster +from matching.keywords import extract_keywords, coverage + +__all__ = [ + "normalize_employer", + "title_score", + "employer_match", + "desc_score", + "cluster", + "extract_keywords", + "coverage", +] \ No newline at end of file diff --git a/packages/matching/src/matching/dedupe.py b/packages/matching/src/matching/dedupe.py new file mode 100644 index 0000000..3463b74 --- /dev/null +++ b/packages/matching/src/matching/dedupe.py @@ -0,0 +1,136 @@ +"""Duplicate clustering for job postings. + +Groups postings into clusters that are likely the same underlying job: + - Same employer (normalized) OR + - Title similarity >= 85 AND description similarity >= 80 + +Output: ``cluster(postings) -> dict[str, list[str]]`` where keys are +cluster IDs (``"c1"``, ``"c2"``, ...) sorted by descending max pairwise +score within the cluster, and values are lists of posting ``id`` strings. +""" + +from __future__ import annotations + +from matching.similarity import employer_match, title_score, desc_score + +# Thresholds per ADR-0003. +TITLE_THRESHOLD = 85.0 +DESC_THRESHOLD = 80.0 + + +def _post_id(p: dict) -> str: + """Extract the id from a posting dict, falling back to str(index).""" + pid = p.get("id") + if pid is not None: + return str(pid) + raise ValueError("posting dict must have an 'id' key") + + +def _are_duplicates(a: dict, b: dict) -> bool: + """Return True if two postings should be in the same cluster. + + Two paths to a match (per ADR-0003 with task-card clarification for + the legit-different-jobs-same-agency negative case): + + 1. Same employer (normalized) AND some content overlap + (title >= 85 OR desc >= 80). + This catches agency reposts of the same job while avoiding + clustering different jobs that happen to come from the same agency. + + 2. Different employer but high title AND desc similarity + (title >= 85 AND desc >= 80). + This catches cross-agency reposts of the same job. + """ + ts = title_score(a.get("title", ""), b.get("title", "")) + ds = desc_score(a.get("description", ""), b.get("description", "")) + same_employer = employer_match(a.get("employer", ""), b.get("employer", "")) + + if same_employer: + # Same employer + at least one content dimension similar. + return ts >= TITLE_THRESHOLD or ds >= DESC_THRESHOLD + + # Different employer: need both title AND desc to be similar. + return ts >= TITLE_THRESHOLD and ds >= DESC_THRESHOLD + + +def _pair_score(a: dict, b: dict) -> float: + """Compute a similarity score between two postings for sorting clusters.""" + ts = title_score(a.get("title", ""), b.get("title", "")) + ds = desc_score(a.get("description", ""), b.get("description", "")) + same_employer = employer_match(a.get("employer", ""), b.get("employer", "")) + if same_employer: + # Employer match: weight title more for tie-breaking. + return 100.0 + ts + return (ts + ds) / 2.0 + + +def cluster(postings: list[dict]) -> dict[str, list[str]]: + """Cluster job postings into duplicate groups. + + Uses union-find so transitive duplicates (A~B, B~C => A~C) are grouped + together. + + Args: + postings: list of dicts with keys ``id``, ``employer``, ``title``, + ``description``. + + Returns: + Dict mapping cluster_id (``"c1"``, ``"c2"``, ...) to a list of + posting id strings. Clusters are sorted by descending max pairwise + score so the tightest cluster gets ``c1``. + """ + n = len(postings) + if n == 0: + return {} + + ids = [_post_id(p) for p in postings] + + # Union-find (disjoint set). + parent: list[int] = list(range(n)) + + def find(x: int) -> int: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[ra] = rb + + # O(n^2) pairwise comparison. + for i in range(n): + for j in range(i + 1, n): + if _are_duplicates(postings[i], postings[j]): + union(i, j) + + # Collect groups. + groups: dict[int, list[int]] = {} + for i in range(n): + root = find(i) + groups.setdefault(root, []).append(i) + + # Compute max pairwise score per group for sorting. + group_scores: list[tuple[float, list[int]]] = [] + for indices in groups.values(): + if len(indices) < 2: + max_sc = 0.0 + else: + max_sc = 0.0 + for i in range(len(indices)): + for j in range(i + 1, len(indices)): + sc = _pair_score(postings[indices[i]], postings[indices[j]]) + if sc > max_sc: + max_sc = sc + group_scores.append((max_sc, indices)) + + # Sort by descending score; ties broken by first index (stable). + group_scores.sort(key=lambda t: (-t[0], t[1][0])) + + # Assign cluster IDs. + result: dict[str, list[str]] = {} + for idx, (_, indices) in enumerate(group_scores, start=1): + result[f"c{idx}"] = [ids[i] for i in indices] + + return result \ No newline at end of file diff --git a/packages/matching/src/matching/keywords.py b/packages/matching/src/matching/keywords.py new file mode 100644 index 0000000..ea86c32 --- /dev/null +++ b/packages/matching/src/matching/keywords.py @@ -0,0 +1,162 @@ +"""Keyword extraction and coverage analysis. + +Deterministic, no LLM. Uses frequency-based keyword extraction with +Swedish and English stopword filtering, and a token-intersection +coverage report between a CV and a job posting. +""" + +from __future__ import annotations + +import re +from collections import Counter + +# --------------------------------------------------------------------------- +# Stopwords (Swedish + English). Conservative lists. +# --------------------------------------------------------------------------- + +_SWEDISH_STOPWORDS: frozenset[str] = frozenset({ + "och", "eller", "som", "att", "den", "det", "de", "vi", "ni", "du", + "han", "hon", "den", "en", "ett", "ar", "har", "var", "var", "inte", + "med", "for", "fran", "till", "pa", "av", "i", "och", "men", "sa", + "när", "då", "hur", "alla", "nagon", "nagot", "alla", "manga", "mycket", + "skall", "ska", "kan", "kommer", "blir", "vore", "vill", "borde", + "efter", "under", "over", "bAKom", "inom", "mellan", "genom", "utan", + "mot", "ut", "fran", "frams", "igår", "idag", "imorgon", +}) + +_ENGLISH_STOPWORDS: frozenset[str] = frozenset({ + "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "being", "have", "has", "had", "do", "does", "did", "will", "would", + "should", "could", "may", "might", "must", "can", "this", "that", + "these", "those", "i", "you", "he", "she", "it", "we", "they", "me", + "him", "her", "us", "them", "my", "your", "his", "its", "our", "their", + "what", "which", "who", "whom", "where", "when", "why", "how", "all", + "each", "every", "both", "few", "more", "most", "other", "some", "such", + "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", + "as", "if", "about", "against", "between", "into", "through", "during", + "before", "after", "above", "below", "up", "down", "out", "off", + "over", "under", "again", "further", "then", "once", "here", "there", + "also", "etc", "e.g", "i.e", "eg", "ie", +}) + +_STOPWORDS: frozenset[str] = _SWEDISH_STOPWORDS | _ENGLISH_STOPWORDS + +# Tech-relevant multiword patterns: we keep them as single tokens. +# e.g. "fast api" -> "fastapi" so it survives as a keyword. +_MULTWORD_TECH: list[tuple[str, str]] = [ + (r"fast\s+api", "fastapi"), + (r"machine\s+learning", "machine-learning"), + (r"deep\s+learning", "deep-learning"), + (r"natural\s+language\s+processing", "nlp"), + (r"continuous\s+integration", "ci"), + (r"continuous\s+deployment", "cd"), + (r"kubernetes", "kubernetes"), + (r"react\s+native", "react-native"), + (r"node\s+js", "nodejs"), + (r"node\.js", "nodejs"), + (r"aws", "aws"), + (r"gcp", "gcp"), + (r"ci/cd", "ci-cd"), +] + +# Minimum token length for keywords. +_MIN_TOKEN_LEN = 2 + +# Tokenizer: split on non-alphanumeric. +_TOKEN_RE = re.compile(r"[a-z0-9+#.\-/]+") + + +def _preprocess_multiwords(text: str) -> str: + """Replace known multiword tech terms with single tokens.""" + result = text.lower() + for pattern, replacement in _MULTWORD_TECH: + result = re.sub(pattern, replacement, result, flags=re.IGNORECASE) + return result + + +def _tokenize(text: str) -> list[str]: + """Tokenize text into lowercased tokens.""" + text = _preprocess_multiwords(text) + raw_tokens = _TOKEN_RE.findall(text.lower()) + tokens: list[str] = [] + for t in raw_tokens: + t = t.strip(".-/") + if not t: + continue + if t in _STOPWORDS: + continue + if len(t) < _MIN_TOKEN_LEN: + continue + # Skip pure numbers (unless they look like versions). + if t.isdigit() and len(t) > 4: + continue + tokens.append(t) + return tokens + + +def extract_keywords(text: str, top_n: int = 30) -> list[str]: + """Extract the top *top_n* keywords from *text* by frequency. + + Keywords are lowercased tokens. Stopwords (Swedish + English) are + removed. Multiword tech terms like "fast api" are collapsed to + "fastapi". + + Args: + text: the text to analyze. + top_n: maximum number of keywords to return. + + Returns: + List of keyword strings, most frequent first. Ties are broken + alphabetically for determinism. + """ + if not text: + return [] + tokens = _tokenize(text) + if not tokens: + return [] + counts: Counter[str] = Counter(tokens) + # Sort by count desc, then alphabetically for deterministic order. + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + return [word for word, _ in ranked[:top_n]] + + +def coverage(cv_text: str, posting_text: str) -> dict[str, list[str] | float]: + """Compute keyword coverage of a CV against a job posting. + + Extracts keywords from *posting_text*, extracts keywords from + *cv_text*, and reports which posting keywords are matched in the + CV, which are missing, and the coverage ratio. + + Args: + cv_text: the candidate's CV text. + posting_text: the job posting text. + + Returns: + Dict with keys: + - ``matched``: list of posting keywords found in CV. + - ``missing``: list of posting keywords NOT found in CV. + - ``ratio``: float (matched / total), 0.0 if no keywords. + """ + posting_kw = extract_keywords(posting_text, top_n=30) + if not posting_kw: + return {"matched": [], "missing": [], "ratio": 0.0} + + cv_kw_set: set[str] = set(extract_keywords(cv_text, top_n=200)) + + matched: list[str] = [] + missing: list[str] = [] + for kw in posting_kw: + if kw in cv_kw_set: + matched.append(kw) + else: + missing.append(kw) + + total = len(posting_kw) + ratio = len(matched) / total if total > 0 else 0.0 + + return { + "matched": matched, + "missing": missing, + "ratio": ratio, + } \ No newline at end of file diff --git a/packages/matching/src/matching/similarity.py b/packages/matching/src/matching/similarity.py new file mode 100644 index 0000000..cb42a0a --- /dev/null +++ b/packages/matching/src/matching/similarity.py @@ -0,0 +1,113 @@ +"""Similarity helpers for job postings. + +Uses rapidfuzz for fuzzy string matching. All functions are deterministic. +""" + +from __future__ import annotations + +import re +from rapidfuzz import fuzz + +# --------------------------------------------------------------------------- +# Agency-suffix / noise words to strip from employer names. +# Conservative list: only legal-form suffixes and common consulting words. +# --------------------------------------------------------------------------- + +_AGENCY_SUFFIXES: list[str] = [ + # Swedish legal forms + "ab", + "aktiebolag", + "hb", + "kb", + "ekonomisk forening", + # English legal forms + "inc", + "corp", + "corporation", + "ltd", + "limited", + "llc", + "gmbh", + "ag", + "sas", + "sarl", + # Consulting / staffing suffixes (agency hints) + "consulting", + "consultancy", + "consult", + "recruitment", + "staffing", + "solutions", + "services", + "group", + "partners", +] + +# Pre-compile regex for trailing suffix removal. +_SUFFIX_RE = re.compile( + r"\s+(" + "|".join(re.escape(s) for s in _AGENCY_SUFFIXES) + r")\.?\s*$", + flags=re.IGNORECASE, +) + +# Characters to collapse: punctuation -> space, then multi-space -> single. +_PUNCT_RE = re.compile(r"[^\w\s]") +_WS_RE = re.compile(r"\s+") + + +def normalize_employer(name: str) -> str: + """Normalize an employer/company name for comparison. + + Steps: + 1. lowercase + 2. strip trailing agency/legal suffixes (ab, consulting, etc.) + 3. remove punctuation + 4. collapse whitespace + + Examples: + >>> normalize_employer("Acme Consulting AB") + 'acme' + >>> normalize_employer("Acme AB") + 'acme' + >>> normalize_employer(" Globex Corp. ") + 'globex' + """ + if not name: + return "" + s = name.strip().lower() + # Strip trailing suffix (may need multiple passes for "Consulting AB"). + for _ in range(3): + new = _SUFFIX_RE.sub("", s) + if new == s: + break + s = new + # Remove punctuation. + s = _PUNCT_RE.sub(" ", s) + s = _WS_RE.sub(" ", s).strip() + return s + + +def title_score(a: str, b: str) -> float: + """Token-set ratio score for two job titles (0-100). + + Uses rapidfuzz ``fuzz.token_set_ratio`` which is order-independent + and handles subsets well. + """ + if not a or not b: + return 0.0 + return float(fuzz.token_set_ratio(a, b)) + + +def employer_match(a: str, b: str) -> bool: + """True if two employer names normalize to the same string.""" + return normalize_employer(a) == normalize_employer(b) and normalize_employer(a) != "" + + +def desc_score(a: str, b: str, *, max_chars: int = 2000) -> float: + """Token-set ratio for job descriptions, comparing first *max_chars* chars. + + Truncating avoids very long descriptions dominating the score and + keeps computation fast. + """ + if not a or not b: + return 0.0 + return float(fuzz.token_set_ratio(a[:max_chars], b[:max_chars])) \ No newline at end of file diff --git a/packages/matching/tests/conftest.py b/packages/matching/tests/conftest.py new file mode 100644 index 0000000..be3db9d --- /dev/null +++ b/packages/matching/tests/conftest.py @@ -0,0 +1,184 @@ +"""Shared fixtures for matching tests. + +Provides agency-repost fixture triples and a legit-different-jobs-same-agency +negative case. +""" + +from __future__ import annotations + +import pytest + + +# --------------------------------------------------------------------------- +# Agency-repost fixture triples. +# Three realistic scenarios where agencies repost the same job. +# Each triple is a list of 3 posting dicts that should all cluster together. +# --------------------------------------------------------------------------- + +# Triple 1: Same role, two agencies + direct employer posting. +# All three have the same title and nearly identical description, but +# different employer names (the two agencies vs the actual company). +TRIPLE_1_AGENCY_REPOST = [ + { + "id": "t1-a", + "employer": "TechCorp AB", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, + { + "id": "t1-b", + "employer": "Nordic IT Consulting AB", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, + { + "id": "t1-c", + "employer": "Acme Recruitment Group", + "title": "Senior Python Developer", + "description": ( + "We are looking for a Senior Python Developer to join our backend team. " + "You will work with Fast API, PostgreSQL, and Docker in a cloud-native " + "environment. 5+ years of Python experience required. " + "Experience with AWS and Kubernetes is a plus." + ), + }, +] + +# Triple 2: Same role reposted by same agency with minor wording variations. +TRIPLE_2_SAME_AGENCY_REPOST = [ + { + "id": "t2-a", + "employer": "Stockholm Tech Staffing AB", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, + { + "id": "t2-b", + "employer": "Stockholm Tech Staffing AB", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, + { + "id": "t2-c", + "employer": "Stockholm Tech Staffing", + "title": "Fullstack Engineer", + "description": ( + "Fullstack Engineer wanted for a fintech startup in Stockholm. " + "Tech stack: React, TypeScript, Node.js, PostgreSQL. " + "You will build customer-facing features and internal tools. " + "Must have experience with CI/CD pipelines." + ), + }, +] + +# Triple 3: Same role, slightly different title but same description body. +# Different employers (agencies), high title + desc similarity. +TRIPLE_3_CROSS_AGENCY = [ + { + "id": "t3-a", + "employer": "Data Recruiting Solutions", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, + { + "id": "t3-b", + "employer": "Cloud Talent Partners", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, + { + "id": "t3-c", + "employer": "Analytics Staffing Ltd", + "title": "Data Engineer", + "description": ( + "We seek a Data Engineer to build and maintain ETL pipelines using " + "Python, Airflow, dbt, and Snowflake. You will design data models, " + "optimize queries, and ensure data quality. Experience with " + "distributed systems and Spark is required." + ), + }, +] + +# --------------------------------------------------------------------------- +# NEGATIVE case: legit different jobs at same agency -- must NOT cluster. +# Same agency employer but different titles and different descriptions. +# --------------------------------------------------------------------------- + +NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY = [ + { + "id": "neg-a", + "employer": "Nordic IT Consulting AB", + "title": "Frontend Developer", + "description": ( + "We are looking for a Frontend Developer with expertise in React " + "and TypeScript. You will build responsive web applications and " + "work closely with our design team. Experience with CSS-in-JS and " + "accessibility standards is required." + ), + }, + { + "id": "neg-b", + "employer": "Nordic IT Consulting AB", + "title": "DevOps Engineer", + "description": ( + "We need a DevOps Engineer to manage our Kubernetes clusters and " + "CI/CD pipelines. You will work with Terraform, ArgoCD, and " + "Prometheus monitoring. Strong Linux and networking background " + "is required. AWS certification is a plus." + ), + }, +] + + +@pytest.fixture +def triple1(): + """Agency repost triple 1: same role, two agencies + employer.""" + return [dict(p) for p in TRIPLE_1_AGENCY_REPOST] + + +@pytest.fixture +def triple2(): + """Agency repost triple 2: same agency reposts same job.""" + return [dict(p) for p in TRIPLE_2_SAME_AGENCY_REPOST] + + +@pytest.fixture +def triple3(): + """Agency repost triple 3: cross-agency same role same description.""" + return [dict(p) for p in TRIPLE_3_CROSS_AGENCY] + + +@pytest.fixture +def negative_same_agency(): + """Negative case: different jobs at same agency, must NOT cluster.""" + return [dict(p) for p in NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY] \ No newline at end of file diff --git a/packages/matching/tests/test_dedupe.py b/packages/matching/tests/test_dedupe.py new file mode 100644 index 0000000..9af1a09 --- /dev/null +++ b/packages/matching/tests/test_dedupe.py @@ -0,0 +1,168 @@ +"""Tests for dedupe clustering.""" + +from __future__ import annotations + +from matching.dedupe import cluster + + +class TestClusterBasic: + def test_empty_list(self): + assert cluster([]) == {} + + def test_single_posting(self): + result = cluster([ + {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"} + ]) + assert len(result) == 1 + assert "c1" in result + assert result["c1"] == ["a"] + + def test_no_duplicates_separate_clusters(self): + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"}, + {"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"}, + {"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"}, + ] + result = cluster(postings) + # Each posting in its own cluster. + total_ids = sum(len(v) for v in result.values()) + assert total_ids == 3 + # All cluster values are singletons. + for ids in result.values(): + assert len(ids) == 1 + + +class TestClusterEmployerMatch: + def test_same_employer_clusters(self): + """Same employer name (normalized) with similar content should cluster.""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, + {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, + ] + result = cluster(postings) + # Same employer + similar title -> same cluster. + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + def test_same_employer_different_content_no_cluster(self): + """Same employer but completely different titles/descriptions should NOT cluster.""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."}, + {"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."}, + ] + result = cluster(postings) + # Same employer but different jobs -> no cluster. + assert all(len(v) == 1 for v in result.values()) + + def test_employer_suffix_variations_cluster(self): + """Acme AB and Acme should cluster (normalized match).""" + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}, + {"id": "b", "employer": "Acme", "title": "Dev", "description": "y"}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + +class TestClusterTitleDescMatch: + def test_title_desc_high_enough(self): + """Different employers but title >= 85 and desc >= 80 -> cluster.""" + desc = ( + "We are looking for a Senior Python Developer to join our backend " + "team. You will work with Fast API, PostgreSQL, and Docker." + ) + postings = [ + {"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc}, + {"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b"} + + def test_title_high_desc_low_no_cluster(self): + """Title similar but desc too different -> no cluster.""" + postings = [ + {"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."}, + {"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."}, + ] + result = cluster(postings) + # Should NOT cluster (different employers, low desc score). + assert len(result) == 2 or all(len(v) == 1 for v in result.values()) + + +class TestClusterTransitive: + def test_transitive_clustering(self): + """If A~B and B~C then A~C should be in same cluster.""" + # A and B same employer + similar title, B and C same employer + similar title. + postings = [ + {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, + {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, + {"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"}, + ] + result = cluster(postings) + assert len(result) == 1 + assert set(result["c1"]) == {"a", "b", "c"} + + +class TestClusterSorting: + def test_cluster_ids_sorted_by_score(self): + """Cluster with higher pairwise score should get c1.""" + # Tight cluster: identical titles and descriptions (different employers). + tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes." + # Looser cluster: different employers, high title but lower desc similarity. + loose_desc_1 = "Python data engineering and pipelines with ETL tools." + loose_desc_2 = "Python data engineering and ETL work with Airflow." + postings = [ + # tight cluster (different employers, high title+desc) + {"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc}, + {"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc}, + # loose cluster (different employers, high title but lower desc) + {"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1}, + {"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2}, + ] + result = cluster(postings) + # Both clusters should exist. + all_ids = set() + for ids in result.values(): + all_ids.update(ids) + assert all_ids == {"t1", "t2", "l1", "l2"} + # c1 should be the tight cluster (higher score: title+desc both 100). + assert set(result["c1"]) == {"t1", "t2"} + + +class TestAgencyRepostTriples: + """Test the 3 agency-repost fixture triples.""" + + def test_triple1_all_cluster(self, triple1): + """Triple 1: 3 postings of same role via different employers cluster.""" + result = cluster(triple1) + assert len(result) == 1 + assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"} + + def test_triple2_all_cluster(self, triple2): + """Triple 2: same agency reposts same job (suffix variations).""" + result = cluster(triple2) + assert len(result) == 1 + assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"} + + def test_triple3_all_cluster(self, triple3): + """Triple 3: cross-agency same role same description.""" + result = cluster(triple3) + assert len(result) == 1 + assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"} + + +class TestNegativeSameAgencyDifferentJobs: + """Negative case: different jobs at same agency must NOT cluster.""" + + def test_different_jobs_same_agency_no_cluster(self, negative_same_agency): + """Different jobs at same agency must NOT cluster. + + Same employer but completely different titles and descriptions. + Per the clustering rule, same employer alone is not sufficient; + some content overlap (title >= 85 OR desc >= 80) is also required. + """ + result = cluster(negative_same_agency) + all_singletons = all(len(v) == 1 for v in result.values()) + assert all_singletons, "Different jobs at same agency should not cluster" \ No newline at end of file diff --git a/packages/matching/tests/test_keywords.py b/packages/matching/tests/test_keywords.py new file mode 100644 index 0000000..0c0c742 --- /dev/null +++ b/packages/matching/tests/test_keywords.py @@ -0,0 +1,152 @@ +"""Tests for keyword extraction and coverage.""" + +from __future__ import annotations + +from matching.keywords import extract_keywords, coverage + + +class TestExtractKeywords: + def test_basic_extraction(self): + text = "Python developer with Fast API experience and PostgreSQL database skills." + kws = extract_keywords(text) + assert "python" in kws + assert "fastapi" in kws + assert "postgresql" in kws + + def test_stopwords_removed(self): + text = "We are looking for a developer with experience in Python." + kws = extract_keywords(text) + assert "we" not in kws + assert "are" not in kws + assert "for" not in kws + assert "a" not in kws + assert "in" not in kws + assert "python" in kws + assert "developer" in kws + + def test_swedish_stopwords_removed(self): + text = "Vi letar efter en Python utvecklare med erfarenhet av Docker." + kws = extract_keywords(text) + assert "vi" not in kws + assert "en" not in kws + assert "av" not in kws + assert "python" in kws + assert "docker" in kws + + def test_top_n_limit(self): + text = "python python python docker docker docker kubernetes kubernetes kubernetes react react react" + kws = extract_keywords(text, top_n=2) + assert len(kws) == 2 + + def test_empty_text(self): + assert extract_keywords("") == [] + + def test_whitespace_only(self): + assert extract_keywords(" ") == [] + + def test_multiword_fastapi(self): + text = "Experience with fast api framework for building REST APIs." + kws = extract_keywords(text) + assert "fastapi" in kws + + def test_multiword_machine_learning(self): + text = "machine learning models for predictive analytics." + kws = extract_keywords(text) + assert "machine-learning" in kws + + def test_frequency_ordering(self): + text = "python python python docker docker kubernetes" + kws = extract_keywords(text, top_n=3) + assert kws[0] == "python" + assert kws[1] == "docker" + assert kws[2] == "kubernetes" + + def test_deterministic_tie_breaking(self): + """Ties in frequency should be broken alphabetically.""" + text = "docker kubernetes" + kws = extract_keywords(text) + # Both have frequency 1, so alphabetical: docker < kubernetes + assert kws[0] == "docker" + assert kws[1] == "kubernetes" + + def test_min_token_length(self): + text = "x y z aa bb cc developer" + kws = extract_keywords(text) + assert "x" not in kws + assert "y" not in kws + assert "z" not in kws + assert "developer" in kws + + def test_tech_terms_preserved(self): + text = "Node.js and React Native for mobile development." + kws = extract_keywords(text) + assert "nodejs" in kws + assert "react-native" in kws + + +class TestCoverage: + def test_full_coverage(self): + cv = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes" + posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes" + result = coverage(cv, posting) + assert result["ratio"] == 1.0 + assert len(result["missing"]) == 0 + + def test_partial_coverage(self): + cv = "Python developer with PostgreSQL and Docker experience" + posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes React" + result = coverage(cv, posting) + assert 0.0 < result["ratio"] < 1.0 + assert "python" in result["matched"] + assert "postgresql" in result["matched"] + assert "docker" in result["matched"] + assert "fastapi" in result["missing"] + assert "kubernetes" in result["missing"] + + def test_zero_coverage(self): + cv = "Chef with experience in French cuisine and menu planning" + posting = "Python developer with Fast API PostgreSQL Docker" + result = coverage(cv, posting) + assert result["ratio"] == 0.0 + assert len(result["matched"]) == 0 + assert len(result["missing"]) > 0 + + def test_empty_posting(self): + result = coverage("Python developer", "") + assert result == {"matched": [], "missing": [], "ratio": 0.0} + + def test_empty_cv(self): + result = coverage("", "Python developer with Docker") + assert result["ratio"] == 0.0 + assert len(result["matched"]) == 0 + assert len(result["missing"]) > 0 + + def test_both_empty(self): + result = coverage("", "") + assert result == {"matched": [], "missing": [], "ratio": 0.0} + + def test_ratio_calculation(self): + cv = "python docker postgresql" + posting = "python docker postgresql kubernetes" + result = coverage(cv, posting) + # 3 of 4 matched (approx, depends on stopword filtering). + assert result["ratio"] > 0.5 + assert result["ratio"] <= 1.0 + + def test_matched_and_missing_lists(self): + cv = "python docker" + posting = "python docker kubernetes react" + result = coverage(cv, posting) + assert "python" in result["matched"] + assert "docker" in result["matched"] + assert "kubernetes" in result["missing"] + assert "react" in result["missing"] + + def test_coverage_returns_dict_keys(self): + result = coverage("python", "python docker") + assert "matched" in result + assert "missing" in result + assert "ratio" in result + assert isinstance(result["matched"], list) + assert isinstance(result["missing"], list) + assert isinstance(result["ratio"], float) \ No newline at end of file diff --git a/packages/matching/tests/test_similarity.py b/packages/matching/tests/test_similarity.py new file mode 100644 index 0000000..9d2fd16 --- /dev/null +++ b/packages/matching/tests/test_similarity.py @@ -0,0 +1,126 @@ +"""Tests for similarity helpers.""" + +from __future__ import annotations + +from matching.similarity import ( + normalize_employer, + title_score, + employer_match, + desc_score, +) + + +class TestNormalizeEmployer: + def test_simple_lowercase(self): + assert normalize_employer("Acme") == "acme" + + def test_strips_swedish_ab(self): + assert normalize_employer("Acme AB") == "acme" + + def test_strips_aktiebolag(self): + assert normalize_employer("Acme Aktiebolag") == "acme" + + def test_strips_consulting_suffix(self): + assert normalize_employer("Nordic IT Consulting AB") == "nordic it" + + def test_strips_corp_suffix(self): + assert normalize_employer("Globex Corp.") == "globex" + + def test_strips_ltd_suffix(self): + assert normalize_employer("Foo Ltd") == "foo" + + def test_strips_recruitment_suffix(self): + assert normalize_employer("Acme Recruitment Group") == "acme" + + def test_strips_multiple_suffixes(self): + # "Consulting AB" should strip both "AB" then "Consulting" + assert normalize_employer("Nordic Consulting AB") == "nordic" + + def test_removes_punctuation(self): + assert normalize_employer("Acme, Inc.") == "acme" + + def test_empty_string(self): + assert normalize_employer("") == "" + + def test_whitespace_only(self): + assert normalize_employer(" ") == "" + + def test_preserves_core_name_with_special_chars(self): + result = normalize_employer("Café Nu AB") + assert "café" in result or "cafe" in result + + def test_dots_in_name_preserved(self): + # Punctuation (except & which gets stripped) is removed; H&M -> h m + result = normalize_employer("H&M AB") + assert result == "h m" + + +class TestTitleScore: + def test_identical_titles(self): + assert title_score("Senior Python Developer", "Senior Python Developer") == 100.0 + + def test_similar_titles_high_score(self): + score = title_score("Python Developer", "Senior Python Developer") + assert score >= 85.0 + + def test_different_titles_low_score(self): + score = title_score("Python Developer", "Frontend Designer") + assert score < 50.0 + + def test_empty_title(self): + assert title_score("", "Something") == 0.0 + + def test_both_empty(self): + assert title_score("", "") == 0.0 + + def test_order_independent(self): + # token_set_ratio is order-independent + a = "Senior Python Developer" + b = "Developer Python Senior" + assert title_score(a, b) == 100.0 + + +class TestEmployerMatch: + def test_same_name_matches(self): + assert employer_match("Acme AB", "Acme AB") is True + + def test_suffix_variation_matches(self): + assert employer_match("Acme AB", "Acme") is True + + def test_different_employers_no_match(self): + assert employer_match("Acme AB", "Globex AB") is False + + def test_consulting_variations_match(self): + assert employer_match("Nordic IT Consulting AB", "Nordic IT") is True + + def test_empty_no_match(self): + assert employer_match("", "") is False + + def test_one_empty_no_match(self): + assert employer_match("Acme", "") is False + + +class TestDescScore: + def test_identical_descriptions(self): + desc = "We are looking for a Python developer with 5 years experience." + assert desc_score(desc, desc) == 100.0 + + def test_similar_descriptions_high(self): + a = "We are looking for a Python developer with 5 years experience." + b = "We are looking for a Python developer with 5 years experience in web." + assert desc_score(a, b) >= 80.0 + + def test_different_descriptions_low(self): + a = "We need a frontend developer skilled in React and CSS." + b = "Looking for a data scientist with Python and SQL expertise." + assert desc_score(a, b) < 50.0 + + def test_empty_desc(self): + assert desc_score("", "something") == 0.0 + + def test_truncation(self): + # Test that truncation to max_chars works. + long_a = "Python " * 1000 + long_b = "Python " * 1000 + score = desc_score(long_a, long_b, max_chars=100) + assert score == 100.0 \ No newline at end of file