Compare commits
No commits in common. "8dceedca2ba148a5d1312481d4b0128bec83e3f2" and "5e539ba713590ac10192d008df779f1221eb8809" have entirely different histories.
8dceedca2b
...
5e539ba713
14 changed files with 0 additions and 1304 deletions
|
|
@ -30,12 +30,9 @@ TASK_CLASS_MAP: dict[str, TaskClass] = {
|
||||||
"score": TaskClass.CHEAP,
|
"score": TaskClass.CHEAP,
|
||||||
"extract": TaskClass.CHEAP,
|
"extract": TaskClass.CHEAP,
|
||||||
"cv_assist": TaskClass.CHEAP,
|
"cv_assist": TaskClass.CHEAP,
|
||||||
"email_classify": TaskClass.CHEAP,
|
|
||||||
"deadline_extract": TaskClass.CHEAP,
|
|
||||||
"cl_critique": TaskClass.STRONG,
|
"cl_critique": TaskClass.STRONG,
|
||||||
"critique": TaskClass.STRONG,
|
"critique": TaskClass.STRONG,
|
||||||
"research": TaskClass.STRONG,
|
"research": TaskClass.STRONG,
|
||||||
"cv_tailor": TaskClass.STRONG,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Default budgets (max output tokens) per task name.
|
# Default budgets (max output tokens) per task name.
|
||||||
|
|
@ -43,12 +40,9 @@ DEFAULT_BUDGETS: dict[str, int] = {
|
||||||
"score": 2000,
|
"score": 2000,
|
||||||
"extract": 4000,
|
"extract": 4000,
|
||||||
"cv_assist": 2000,
|
"cv_assist": 2000,
|
||||||
"email_classify": 1000,
|
|
||||||
"deadline_extract": 500,
|
|
||||||
"cl_critique": 4000,
|
"cl_critique": 4000,
|
||||||
"critique": 6000,
|
"critique": 6000,
|
||||||
"research": 4000,
|
"research": 4000,
|
||||||
"cv_tailor": 6000,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -59,41 +59,6 @@ MOCK_OUTPUTS: dict[str, dict] = {
|
||||||
"summary": "The company is a mid-size tech firm focused on cloud infrastructure.",
|
"summary": "The company is a mid-size tech firm focused on cloud infrastructure.",
|
||||||
"key_points": ["Founded in 2015", "Series B funding", "Remote-first culture"],
|
"key_points": ["Founded in 2015", "Series B funding", "Remote-first culture"],
|
||||||
},
|
},
|
||||||
"email_classify": {
|
|
||||||
"classification": "interview_invite",
|
|
||||||
"state_proposal": "interviewing",
|
|
||||||
"reason": "The email contains an invitation to schedule an interview.",
|
|
||||||
},
|
|
||||||
"cv_tailor": {
|
|
||||||
"tailored_cv": {
|
|
||||||
"summary": "Senior Python Developer with 6+ years building scalable backend systems.",
|
|
||||||
"skills": [
|
|
||||||
"Python",
|
|
||||||
"Fast API",
|
|
||||||
"PostgreSQL",
|
|
||||||
"Docker",
|
|
||||||
"Kubernetes",
|
|
||||||
"AWS",
|
|
||||||
],
|
|
||||||
"experience": [
|
|
||||||
{
|
|
||||||
"company": "TechCorp",
|
|
||||||
"role": "Senior Backend Engineer",
|
|
||||||
"bullets": [
|
|
||||||
"Led migration of monolith to microservices using Fast API",
|
|
||||||
"Reduced API latency by 40% through query optimization and caching",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"change_log": [
|
|
||||||
{"action": "reordered", "detail": "Moved Python and Fast API to top of skills"},
|
|
||||||
{"action": "rephrased", "detail": "Rewrote first experience bullet to emphasize Fast API"},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"deadline_extract": {
|
|
||||||
"apply_by": None,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Default mock output for unknown task names.
|
# Default mock output for unknown task names.
|
||||||
|
|
|
||||||
|
|
@ -270,11 +270,8 @@ class TestGatewayConfig:
|
||||||
assert config.get_task_class("score") == TaskClass.CHEAP
|
assert config.get_task_class("score") == TaskClass.CHEAP
|
||||||
assert config.get_task_class("extract") == TaskClass.CHEAP
|
assert config.get_task_class("extract") == TaskClass.CHEAP
|
||||||
assert config.get_task_class("cv_assist") == TaskClass.CHEAP
|
assert config.get_task_class("cv_assist") == TaskClass.CHEAP
|
||||||
assert config.get_task_class("email_classify") == TaskClass.CHEAP
|
|
||||||
assert config.get_task_class("deadline_extract") == TaskClass.CHEAP
|
|
||||||
assert config.get_task_class("critique") == TaskClass.STRONG
|
assert config.get_task_class("critique") == TaskClass.STRONG
|
||||||
assert config.get_task_class("cl_critique") == TaskClass.STRONG
|
assert config.get_task_class("cl_critique") == TaskClass.STRONG
|
||||||
assert config.get_task_class("cv_tailor") == TaskClass.STRONG
|
|
||||||
|
|
||||||
def test_get_model_routing(self) -> None:
|
def test_get_model_routing(self) -> None:
|
||||||
config = mock_config(cheap_model="cheap-model", strong_model="strong-model")
|
config = mock_config(cheap_model="cheap-model", strong_model="strong-model")
|
||||||
|
|
|
||||||
|
|
@ -1,125 +0,0 @@
|
||||||
"""Tests for new v1.1 mock tasks: email_classify, cv_tailor, deadline_extract."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from llm_gateway.config import GatewayConfig, ProviderConfig, TaskClass
|
|
||||||
from llm_gateway.gateway import Gateway
|
|
||||||
from llm_gateway.mock import get_mock_output, MOCK_OUTPUTS
|
|
||||||
|
|
||||||
|
|
||||||
def mock_config(**overrides) -> GatewayConfig:
|
|
||||||
"""Build a config in mock mode (no API key)."""
|
|
||||||
primary = ProviderConfig(
|
|
||||||
name="primary",
|
|
||||||
base_url="https://mock.example.com/v1",
|
|
||||||
api_key="",
|
|
||||||
model="glm-5.2",
|
|
||||||
)
|
|
||||||
defaults = {
|
|
||||||
"primary": primary,
|
|
||||||
"fallback": None,
|
|
||||||
"cheap_model": "glm-5.2",
|
|
||||||
"strong_model": "glm-5.2",
|
|
||||||
"budgets": {
|
|
||||||
"score": 2000,
|
|
||||||
"extract": 4000,
|
|
||||||
"email_classify": 1000,
|
|
||||||
"deadline_extract": 500,
|
|
||||||
"cv_tailor": 6000,
|
|
||||||
"default": 4000,
|
|
||||||
},
|
|
||||||
"max_retries": 2,
|
|
||||||
}
|
|
||||||
defaults.update(overrides)
|
|
||||||
return GatewayConfig(**defaults)
|
|
||||||
|
|
||||||
|
|
||||||
class TestEmailClassifyMock:
|
|
||||||
async def test_email_classify_returns_deterministic(self) -> None:
|
|
||||||
"""email_classify mock returns interview_invite classification."""
|
|
||||||
config = mock_config()
|
|
||||||
gw = Gateway(config)
|
|
||||||
result_a = await gw.run_task("email_classify", "Email from recruiter")
|
|
||||||
result_b = await gw.run_task("email_classify", "Email from recruiter")
|
|
||||||
assert result_a == result_b
|
|
||||||
assert result_a["classification"] == "interview_invite"
|
|
||||||
assert result_a["state_proposal"] == "interviewing"
|
|
||||||
assert "reason" in result_a
|
|
||||||
await gw.aclose()
|
|
||||||
|
|
||||||
async def test_email_classify_is_cheap(self) -> None:
|
|
||||||
"""email_classify should be classified as CHEAP."""
|
|
||||||
config = mock_config()
|
|
||||||
assert config.get_task_class("email_classify") == TaskClass.CHEAP
|
|
||||||
assert config.get_model("email_classify") == config.cheap_model
|
|
||||||
|
|
||||||
def test_email_classify_in_mock_outputs(self) -> None:
|
|
||||||
"""email_classify should be in MOCK_OUTPUTS."""
|
|
||||||
assert "email_classify" in MOCK_OUTPUTS
|
|
||||||
output = get_mock_output("email_classify")
|
|
||||||
assert output["classification"] == "interview_invite"
|
|
||||||
assert output["state_proposal"] == "interviewing"
|
|
||||||
|
|
||||||
|
|
||||||
class TestCvTailorMock:
|
|
||||||
async def test_cv_tailor_returns_deterministic(self) -> None:
|
|
||||||
"""cv_tailor mock returns tailored CV with change_log."""
|
|
||||||
config = mock_config()
|
|
||||||
gw = Gateway(config)
|
|
||||||
result_a = await gw.run_task("cv_tailor", "Tailor CV for posting")
|
|
||||||
result_b = await gw.run_task("cv_tailor", "Tailor CV for posting")
|
|
||||||
assert result_a == result_b
|
|
||||||
assert "tailored_cv" in result_a
|
|
||||||
assert "change_log" in result_a
|
|
||||||
assert isinstance(result_a["change_log"], list)
|
|
||||||
assert len(result_a["change_log"]) >= 1
|
|
||||||
# Check change_log entries have action and detail.
|
|
||||||
for entry in result_a["change_log"]:
|
|
||||||
assert "action" in entry
|
|
||||||
assert "detail" in entry
|
|
||||||
await gw.aclose()
|
|
||||||
|
|
||||||
async def test_cv_tailor_is_strong(self) -> None:
|
|
||||||
"""cv_tailor should be classified as STRONG."""
|
|
||||||
config = mock_config()
|
|
||||||
assert config.get_task_class("cv_tailor") == TaskClass.STRONG
|
|
||||||
assert config.get_model("cv_tailor") == config.strong_model
|
|
||||||
|
|
||||||
def test_cv_tailor_in_mock_outputs(self) -> None:
|
|
||||||
"""cv_tailor should be in MOCK_OUTPUTS."""
|
|
||||||
assert "cv_tailor" in MOCK_OUTPUTS
|
|
||||||
output = get_mock_output("cv_tailor")
|
|
||||||
assert "tailored_cv" in output
|
|
||||||
assert "change_log" in output
|
|
||||||
# Check it has skills and experience.
|
|
||||||
assert "skills" in output["tailored_cv"]
|
|
||||||
assert "experience" in output["tailored_cv"]
|
|
||||||
|
|
||||||
|
|
||||||
class TestDeadlineExtractMock:
|
|
||||||
async def test_deadline_extract_returns_deterministic(self) -> None:
|
|
||||||
"""deadline_extract mock returns apply_by (null by default)."""
|
|
||||||
config = mock_config()
|
|
||||||
gw = Gateway(config)
|
|
||||||
result_a = await gw.run_task("deadline_extract", "Extract deadline")
|
|
||||||
result_b = await gw.run_task("deadline_extract", "Extract deadline")
|
|
||||||
assert result_a == result_b
|
|
||||||
assert "apply_by" in result_a
|
|
||||||
# Default mock has null deadline.
|
|
||||||
assert result_a["apply_by"] is None
|
|
||||||
await gw.aclose()
|
|
||||||
|
|
||||||
async def test_deadline_extract_is_cheap(self) -> None:
|
|
||||||
"""deadline_extract should be classified as CHEAP."""
|
|
||||||
config = mock_config()
|
|
||||||
assert config.get_task_class("deadline_extract") == TaskClass.CHEAP
|
|
||||||
assert config.get_model("deadline_extract") == config.cheap_model
|
|
||||||
|
|
||||||
def test_deadline_extract_in_mock_outputs(self) -> None:
|
|
||||||
"""deadline_extract should be in MOCK_OUTPUTS."""
|
|
||||||
assert "deadline_extract" in MOCK_OUTPUTS
|
|
||||||
output = get_mock_output("deadline_extract")
|
|
||||||
assert "apply_by" in output
|
|
||||||
assert output["apply_by"] is None
|
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
# packages/matching
|
|
||||||
|
|
||||||
Job posting similarity, dedupe clustering, and keyword coverage for the
|
|
||||||
jobhunt-platform v1.1 agency duplicate detection feature (ADR-0003).
|
|
||||||
|
|
||||||
## Modules
|
|
||||||
|
|
||||||
### similarity.py
|
|
||||||
- `normalize_employer(name)` -- lowercase, strip agency/legal suffixes (AB, Consulting, etc.), remove punctuation.
|
|
||||||
- `title_score(a, b)` -- rapidfuzz token_set_ratio on job titles (0-100).
|
|
||||||
- `employer_match(a, b)` -- True if normalized employer names are equal.
|
|
||||||
- `desc_score(a, b, max_chars=2000)` -- token_set_ratio on first 2000 chars of descriptions.
|
|
||||||
|
|
||||||
### dedupe.py
|
|
||||||
- `cluster(postings: list[dict]) -> dict[str, list[str]]` -- group postings into duplicate clusters.
|
|
||||||
|
|
||||||
Clustering rule (per ADR-0003):
|
|
||||||
- Same employer (normalized) **OR**
|
|
||||||
- Title similarity >= 85 **AND** description similarity >= 80
|
|
||||||
|
|
||||||
Uses union-find for transitive grouping. Clusters are sorted by descending
|
|
||||||
max pairwise score (`c1` = tightest cluster).
|
|
||||||
|
|
||||||
### keywords.py
|
|
||||||
- `extract_keywords(text, top_n=30)` -- frequency-based keyword extraction with Swedish + English stopword removal.
|
|
||||||
- `coverage(cv_text, posting_text)` -- computes keyword coverage of a CV against a job posting.
|
|
||||||
|
|
||||||
Multiword tech terms like "fast api" are collapsed to "fastapi" so they survive as single keywords.
|
|
||||||
|
|
||||||
## Installation (uv)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd packages/matching
|
|
||||||
uv venv && source .venv/bin/activate
|
|
||||||
uv pip install -e ".[dev]"
|
|
||||||
pytest
|
|
||||||
```
|
|
||||||
|
|
||||||
## License
|
|
||||||
MIT
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
[project]
|
|
||||||
name = "matching"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Job posting similarity, dedupe clustering, and keyword coverage for agency duplicate detection."
|
|
||||||
requires-python = ">=3.13"
|
|
||||||
dependencies = [
|
|
||||||
"rapidfuzz>=3.6",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
|
||||||
dev = [
|
|
||||||
"pytest>=8.0",
|
|
||||||
"pytest-asyncio>=0.24",
|
|
||||||
]
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/matching"]
|
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
|
||||||
testpaths = ["tests"]
|
|
||||||
pythonpath = ["src"]
|
|
||||||
asyncio_mode = "auto"
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
"""Job posting matching package.
|
|
||||||
|
|
||||||
Provides:
|
|
||||||
- similarity: normalized text comparison (employer match, title/desc scores)
|
|
||||||
- dedupe: cluster postings into duplicate groups
|
|
||||||
- keywords: keyword extraction and CV-vs-posting coverage
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from matching.similarity import (
|
|
||||||
normalize_employer,
|
|
||||||
title_score,
|
|
||||||
employer_match,
|
|
||||||
desc_score,
|
|
||||||
)
|
|
||||||
from matching.dedupe import cluster
|
|
||||||
from matching.keywords import extract_keywords, coverage
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"normalize_employer",
|
|
||||||
"title_score",
|
|
||||||
"employer_match",
|
|
||||||
"desc_score",
|
|
||||||
"cluster",
|
|
||||||
"extract_keywords",
|
|
||||||
"coverage",
|
|
||||||
]
|
|
||||||
|
|
@ -1,136 +0,0 @@
|
||||||
"""Duplicate clustering for job postings.
|
|
||||||
|
|
||||||
Groups postings into clusters that are likely the same underlying job:
|
|
||||||
- Same employer (normalized) OR
|
|
||||||
- Title similarity >= 85 AND description similarity >= 80
|
|
||||||
|
|
||||||
Output: ``cluster(postings) -> dict[str, list[str]]`` where keys are
|
|
||||||
cluster IDs (``"c1"``, ``"c2"``, ...) sorted by descending max pairwise
|
|
||||||
score within the cluster, and values are lists of posting ``id`` strings.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from matching.similarity import employer_match, title_score, desc_score
|
|
||||||
|
|
||||||
# Thresholds per ADR-0003.
|
|
||||||
TITLE_THRESHOLD = 85.0
|
|
||||||
DESC_THRESHOLD = 80.0
|
|
||||||
|
|
||||||
|
|
||||||
def _post_id(p: dict) -> str:
|
|
||||||
"""Extract the id from a posting dict, falling back to str(index)."""
|
|
||||||
pid = p.get("id")
|
|
||||||
if pid is not None:
|
|
||||||
return str(pid)
|
|
||||||
raise ValueError("posting dict must have an 'id' key")
|
|
||||||
|
|
||||||
|
|
||||||
def _are_duplicates(a: dict, b: dict) -> bool:
|
|
||||||
"""Return True if two postings should be in the same cluster.
|
|
||||||
|
|
||||||
Two paths to a match (per ADR-0003 with task-card clarification for
|
|
||||||
the legit-different-jobs-same-agency negative case):
|
|
||||||
|
|
||||||
1. Same employer (normalized) AND some content overlap
|
|
||||||
(title >= 85 OR desc >= 80).
|
|
||||||
This catches agency reposts of the same job while avoiding
|
|
||||||
clustering different jobs that happen to come from the same agency.
|
|
||||||
|
|
||||||
2. Different employer but high title AND desc similarity
|
|
||||||
(title >= 85 AND desc >= 80).
|
|
||||||
This catches cross-agency reposts of the same job.
|
|
||||||
"""
|
|
||||||
ts = title_score(a.get("title", ""), b.get("title", ""))
|
|
||||||
ds = desc_score(a.get("description", ""), b.get("description", ""))
|
|
||||||
same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
|
|
||||||
|
|
||||||
if same_employer:
|
|
||||||
# Same employer + at least one content dimension similar.
|
|
||||||
return ts >= TITLE_THRESHOLD or ds >= DESC_THRESHOLD
|
|
||||||
|
|
||||||
# Different employer: need both title AND desc to be similar.
|
|
||||||
return ts >= TITLE_THRESHOLD and ds >= DESC_THRESHOLD
|
|
||||||
|
|
||||||
|
|
||||||
def _pair_score(a: dict, b: dict) -> float:
|
|
||||||
"""Compute a similarity score between two postings for sorting clusters."""
|
|
||||||
ts = title_score(a.get("title", ""), b.get("title", ""))
|
|
||||||
ds = desc_score(a.get("description", ""), b.get("description", ""))
|
|
||||||
same_employer = employer_match(a.get("employer", ""), b.get("employer", ""))
|
|
||||||
if same_employer:
|
|
||||||
# Employer match: weight title more for tie-breaking.
|
|
||||||
return 100.0 + ts
|
|
||||||
return (ts + ds) / 2.0
|
|
||||||
|
|
||||||
|
|
||||||
def cluster(postings: list[dict]) -> dict[str, list[str]]:
|
|
||||||
"""Cluster job postings into duplicate groups.
|
|
||||||
|
|
||||||
Uses union-find so transitive duplicates (A~B, B~C => A~C) are grouped
|
|
||||||
together.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
postings: list of dicts with keys ``id``, ``employer``, ``title``,
|
|
||||||
``description``.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict mapping cluster_id (``"c1"``, ``"c2"``, ...) to a list of
|
|
||||||
posting id strings. Clusters are sorted by descending max pairwise
|
|
||||||
score so the tightest cluster gets ``c1``.
|
|
||||||
"""
|
|
||||||
n = len(postings)
|
|
||||||
if n == 0:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
ids = [_post_id(p) for p in postings]
|
|
||||||
|
|
||||||
# Union-find (disjoint set).
|
|
||||||
parent: list[int] = list(range(n))
|
|
||||||
|
|
||||||
def find(x: int) -> int:
|
|
||||||
while parent[x] != x:
|
|
||||||
parent[x] = parent[parent[x]]
|
|
||||||
x = parent[x]
|
|
||||||
return x
|
|
||||||
|
|
||||||
def union(a: int, b: int) -> None:
|
|
||||||
ra, rb = find(a), find(b)
|
|
||||||
if ra != rb:
|
|
||||||
parent[ra] = rb
|
|
||||||
|
|
||||||
# O(n^2) pairwise comparison.
|
|
||||||
for i in range(n):
|
|
||||||
for j in range(i + 1, n):
|
|
||||||
if _are_duplicates(postings[i], postings[j]):
|
|
||||||
union(i, j)
|
|
||||||
|
|
||||||
# Collect groups.
|
|
||||||
groups: dict[int, list[int]] = {}
|
|
||||||
for i in range(n):
|
|
||||||
root = find(i)
|
|
||||||
groups.setdefault(root, []).append(i)
|
|
||||||
|
|
||||||
# Compute max pairwise score per group for sorting.
|
|
||||||
group_scores: list[tuple[float, list[int]]] = []
|
|
||||||
for indices in groups.values():
|
|
||||||
if len(indices) < 2:
|
|
||||||
max_sc = 0.0
|
|
||||||
else:
|
|
||||||
max_sc = 0.0
|
|
||||||
for i in range(len(indices)):
|
|
||||||
for j in range(i + 1, len(indices)):
|
|
||||||
sc = _pair_score(postings[indices[i]], postings[indices[j]])
|
|
||||||
if sc > max_sc:
|
|
||||||
max_sc = sc
|
|
||||||
group_scores.append((max_sc, indices))
|
|
||||||
|
|
||||||
# Sort by descending score; ties broken by first index (stable).
|
|
||||||
group_scores.sort(key=lambda t: (-t[0], t[1][0]))
|
|
||||||
|
|
||||||
# Assign cluster IDs.
|
|
||||||
result: dict[str, list[str]] = {}
|
|
||||||
for idx, (_, indices) in enumerate(group_scores, start=1):
|
|
||||||
result[f"c{idx}"] = [ids[i] for i in indices]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
@ -1,162 +0,0 @@
|
||||||
"""Keyword extraction and coverage analysis.
|
|
||||||
|
|
||||||
Deterministic, no LLM. Uses frequency-based keyword extraction with
|
|
||||||
Swedish and English stopword filtering, and a token-intersection
|
|
||||||
coverage report between a CV and a job posting.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from collections import Counter
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Stopwords (Swedish + English). Conservative lists.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
_SWEDISH_STOPWORDS: frozenset[str] = frozenset({
|
|
||||||
"och", "eller", "som", "att", "den", "det", "de", "vi", "ni", "du",
|
|
||||||
"han", "hon", "den", "en", "ett", "ar", "har", "var", "var", "inte",
|
|
||||||
"med", "for", "fran", "till", "pa", "av", "i", "och", "men", "sa",
|
|
||||||
"när", "då", "hur", "alla", "nagon", "nagot", "alla", "manga", "mycket",
|
|
||||||
"skall", "ska", "kan", "kommer", "blir", "vore", "vill", "borde",
|
|
||||||
"efter", "under", "over", "bAKom", "inom", "mellan", "genom", "utan",
|
|
||||||
"mot", "ut", "fran", "frams", "igår", "idag", "imorgon",
|
|
||||||
})
|
|
||||||
|
|
||||||
_ENGLISH_STOPWORDS: frozenset[str] = frozenset({
|
|
||||||
"a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
|
|
||||||
"of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
|
|
||||||
"being", "have", "has", "had", "do", "does", "did", "will", "would",
|
|
||||||
"should", "could", "may", "might", "must", "can", "this", "that",
|
|
||||||
"these", "those", "i", "you", "he", "she", "it", "we", "they", "me",
|
|
||||||
"him", "her", "us", "them", "my", "your", "his", "its", "our", "their",
|
|
||||||
"what", "which", "who", "whom", "where", "when", "why", "how", "all",
|
|
||||||
"each", "every", "both", "few", "more", "most", "other", "some", "such",
|
|
||||||
"no", "nor", "not", "only", "own", "same", "so", "than", "too", "very",
|
|
||||||
"as", "if", "about", "against", "between", "into", "through", "during",
|
|
||||||
"before", "after", "above", "below", "up", "down", "out", "off",
|
|
||||||
"over", "under", "again", "further", "then", "once", "here", "there",
|
|
||||||
"also", "etc", "e.g", "i.e", "eg", "ie",
|
|
||||||
})
|
|
||||||
|
|
||||||
_STOPWORDS: frozenset[str] = _SWEDISH_STOPWORDS | _ENGLISH_STOPWORDS
|
|
||||||
|
|
||||||
# Tech-relevant multiword patterns: we keep them as single tokens.
|
|
||||||
# e.g. "fast api" -> "fastapi" so it survives as a keyword.
|
|
||||||
_MULTWORD_TECH: list[tuple[str, str]] = [
|
|
||||||
(r"fast\s+api", "fastapi"),
|
|
||||||
(r"machine\s+learning", "machine-learning"),
|
|
||||||
(r"deep\s+learning", "deep-learning"),
|
|
||||||
(r"natural\s+language\s+processing", "nlp"),
|
|
||||||
(r"continuous\s+integration", "ci"),
|
|
||||||
(r"continuous\s+deployment", "cd"),
|
|
||||||
(r"kubernetes", "kubernetes"),
|
|
||||||
(r"react\s+native", "react-native"),
|
|
||||||
(r"node\s+js", "nodejs"),
|
|
||||||
(r"node\.js", "nodejs"),
|
|
||||||
(r"aws", "aws"),
|
|
||||||
(r"gcp", "gcp"),
|
|
||||||
(r"ci/cd", "ci-cd"),
|
|
||||||
]
|
|
||||||
|
|
||||||
# Minimum token length for keywords.
|
|
||||||
_MIN_TOKEN_LEN = 2
|
|
||||||
|
|
||||||
# Tokenizer: split on non-alphanumeric.
|
|
||||||
_TOKEN_RE = re.compile(r"[a-z0-9+#.\-/]+")
|
|
||||||
|
|
||||||
|
|
||||||
def _preprocess_multiwords(text: str) -> str:
|
|
||||||
"""Replace known multiword tech terms with single tokens."""
|
|
||||||
result = text.lower()
|
|
||||||
for pattern, replacement in _MULTWORD_TECH:
|
|
||||||
result = re.sub(pattern, replacement, result, flags=re.IGNORECASE)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _tokenize(text: str) -> list[str]:
|
|
||||||
"""Tokenize text into lowercased tokens."""
|
|
||||||
text = _preprocess_multiwords(text)
|
|
||||||
raw_tokens = _TOKEN_RE.findall(text.lower())
|
|
||||||
tokens: list[str] = []
|
|
||||||
for t in raw_tokens:
|
|
||||||
t = t.strip(".-/")
|
|
||||||
if not t:
|
|
||||||
continue
|
|
||||||
if t in _STOPWORDS:
|
|
||||||
continue
|
|
||||||
if len(t) < _MIN_TOKEN_LEN:
|
|
||||||
continue
|
|
||||||
# Skip pure numbers (unless they look like versions).
|
|
||||||
if t.isdigit() and len(t) > 4:
|
|
||||||
continue
|
|
||||||
tokens.append(t)
|
|
||||||
return tokens
|
|
||||||
|
|
||||||
|
|
||||||
def extract_keywords(text: str, top_n: int = 30) -> list[str]:
|
|
||||||
"""Extract the top *top_n* keywords from *text* by frequency.
|
|
||||||
|
|
||||||
Keywords are lowercased tokens. Stopwords (Swedish + English) are
|
|
||||||
removed. Multiword tech terms like "fast api" are collapsed to
|
|
||||||
"fastapi".
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: the text to analyze.
|
|
||||||
top_n: maximum number of keywords to return.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of keyword strings, most frequent first. Ties are broken
|
|
||||||
alphabetically for determinism.
|
|
||||||
"""
|
|
||||||
if not text:
|
|
||||||
return []
|
|
||||||
tokens = _tokenize(text)
|
|
||||||
if not tokens:
|
|
||||||
return []
|
|
||||||
counts: Counter[str] = Counter(tokens)
|
|
||||||
# Sort by count desc, then alphabetically for deterministic order.
|
|
||||||
ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
||||||
return [word for word, _ in ranked[:top_n]]
|
|
||||||
|
|
||||||
|
|
||||||
def coverage(cv_text: str, posting_text: str) -> dict[str, list[str] | float]:
|
|
||||||
"""Compute keyword coverage of a CV against a job posting.
|
|
||||||
|
|
||||||
Extracts keywords from *posting_text*, extracts keywords from
|
|
||||||
*cv_text*, and reports which posting keywords are matched in the
|
|
||||||
CV, which are missing, and the coverage ratio.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cv_text: the candidate's CV text.
|
|
||||||
posting_text: the job posting text.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with keys:
|
|
||||||
- ``matched``: list of posting keywords found in CV.
|
|
||||||
- ``missing``: list of posting keywords NOT found in CV.
|
|
||||||
- ``ratio``: float (matched / total), 0.0 if no keywords.
|
|
||||||
"""
|
|
||||||
posting_kw = extract_keywords(posting_text, top_n=30)
|
|
||||||
if not posting_kw:
|
|
||||||
return {"matched": [], "missing": [], "ratio": 0.0}
|
|
||||||
|
|
||||||
cv_kw_set: set[str] = set(extract_keywords(cv_text, top_n=200))
|
|
||||||
|
|
||||||
matched: list[str] = []
|
|
||||||
missing: list[str] = []
|
|
||||||
for kw in posting_kw:
|
|
||||||
if kw in cv_kw_set:
|
|
||||||
matched.append(kw)
|
|
||||||
else:
|
|
||||||
missing.append(kw)
|
|
||||||
|
|
||||||
total = len(posting_kw)
|
|
||||||
ratio = len(matched) / total if total > 0 else 0.0
|
|
||||||
|
|
||||||
return {
|
|
||||||
"matched": matched,
|
|
||||||
"missing": missing,
|
|
||||||
"ratio": ratio,
|
|
||||||
}
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
"""Similarity helpers for job postings.
|
|
||||||
|
|
||||||
Uses rapidfuzz for fuzzy string matching. All functions are deterministic.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from rapidfuzz import fuzz
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Agency-suffix / noise words to strip from employer names.
|
|
||||||
# Conservative list: only legal-form suffixes and common consulting words.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
_AGENCY_SUFFIXES: list[str] = [
|
|
||||||
# Swedish legal forms
|
|
||||||
"ab",
|
|
||||||
"aktiebolag",
|
|
||||||
"hb",
|
|
||||||
"kb",
|
|
||||||
"ekonomisk forening",
|
|
||||||
# English legal forms
|
|
||||||
"inc",
|
|
||||||
"corp",
|
|
||||||
"corporation",
|
|
||||||
"ltd",
|
|
||||||
"limited",
|
|
||||||
"llc",
|
|
||||||
"gmbh",
|
|
||||||
"ag",
|
|
||||||
"sas",
|
|
||||||
"sarl",
|
|
||||||
# Consulting / staffing suffixes (agency hints)
|
|
||||||
"consulting",
|
|
||||||
"consultancy",
|
|
||||||
"consult",
|
|
||||||
"recruitment",
|
|
||||||
"staffing",
|
|
||||||
"solutions",
|
|
||||||
"services",
|
|
||||||
"group",
|
|
||||||
"partners",
|
|
||||||
]
|
|
||||||
|
|
||||||
# Pre-compile regex for trailing suffix removal.
|
|
||||||
_SUFFIX_RE = re.compile(
|
|
||||||
r"\s+(" + "|".join(re.escape(s) for s in _AGENCY_SUFFIXES) + r")\.?\s*$",
|
|
||||||
flags=re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Characters to collapse: punctuation -> space, then multi-space -> single.
|
|
||||||
_PUNCT_RE = re.compile(r"[^\w\s]")
|
|
||||||
_WS_RE = re.compile(r"\s+")
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_employer(name: str) -> str:
|
|
||||||
"""Normalize an employer/company name for comparison.
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. lowercase
|
|
||||||
2. strip trailing agency/legal suffixes (ab, consulting, etc.)
|
|
||||||
3. remove punctuation
|
|
||||||
4. collapse whitespace
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
>>> normalize_employer("Acme Consulting AB")
|
|
||||||
'acme'
|
|
||||||
>>> normalize_employer("Acme AB")
|
|
||||||
'acme'
|
|
||||||
>>> normalize_employer(" Globex Corp. ")
|
|
||||||
'globex'
|
|
||||||
"""
|
|
||||||
if not name:
|
|
||||||
return ""
|
|
||||||
s = name.strip().lower()
|
|
||||||
# Strip trailing suffix (may need multiple passes for "Consulting AB").
|
|
||||||
for _ in range(3):
|
|
||||||
new = _SUFFIX_RE.sub("", s)
|
|
||||||
if new == s:
|
|
||||||
break
|
|
||||||
s = new
|
|
||||||
# Remove punctuation.
|
|
||||||
s = _PUNCT_RE.sub(" ", s)
|
|
||||||
s = _WS_RE.sub(" ", s).strip()
|
|
||||||
return s
|
|
||||||
|
|
||||||
|
|
||||||
def title_score(a: str, b: str) -> float:
|
|
||||||
"""Token-set ratio score for two job titles (0-100).
|
|
||||||
|
|
||||||
Uses rapidfuzz ``fuzz.token_set_ratio`` which is order-independent
|
|
||||||
and handles subsets well.
|
|
||||||
"""
|
|
||||||
if not a or not b:
|
|
||||||
return 0.0
|
|
||||||
return float(fuzz.token_set_ratio(a, b))
|
|
||||||
|
|
||||||
|
|
||||||
def employer_match(a: str, b: str) -> bool:
|
|
||||||
"""True if two employer names normalize to the same string."""
|
|
||||||
return normalize_employer(a) == normalize_employer(b) and normalize_employer(a) != ""
|
|
||||||
|
|
||||||
|
|
||||||
def desc_score(a: str, b: str, *, max_chars: int = 2000) -> float:
|
|
||||||
"""Token-set ratio for job descriptions, comparing first *max_chars* chars.
|
|
||||||
|
|
||||||
Truncating avoids very long descriptions dominating the score and
|
|
||||||
keeps computation fast.
|
|
||||||
"""
|
|
||||||
if not a or not b:
|
|
||||||
return 0.0
|
|
||||||
return float(fuzz.token_set_ratio(a[:max_chars], b[:max_chars]))
|
|
||||||
|
|
@ -1,184 +0,0 @@
|
||||||
"""Shared fixtures for matching tests.
|
|
||||||
|
|
||||||
Provides agency-repost fixture triples and a legit-different-jobs-same-agency
|
|
||||||
negative case.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Agency-repost fixture triples.
|
|
||||||
# Three realistic scenarios where agencies repost the same job.
|
|
||||||
# Each triple is a list of 3 posting dicts that should all cluster together.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
# Triple 1: Same role, two agencies + direct employer posting.
|
|
||||||
# All three have the same title and nearly identical description, but
|
|
||||||
# different employer names (the two agencies vs the actual company).
|
|
||||||
TRIPLE_1_AGENCY_REPOST = [
|
|
||||||
{
|
|
||||||
"id": "t1-a",
|
|
||||||
"employer": "TechCorp AB",
|
|
||||||
"title": "Senior Python Developer",
|
|
||||||
"description": (
|
|
||||||
"We are looking for a Senior Python Developer to join our backend team. "
|
|
||||||
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
|
|
||||||
"environment. 5+ years of Python experience required. "
|
|
||||||
"Experience with AWS and Kubernetes is a plus."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t1-b",
|
|
||||||
"employer": "Nordic IT Consulting AB",
|
|
||||||
"title": "Senior Python Developer",
|
|
||||||
"description": (
|
|
||||||
"We are looking for a Senior Python Developer to join our backend team. "
|
|
||||||
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
|
|
||||||
"environment. 5+ years of Python experience required. "
|
|
||||||
"Experience with AWS and Kubernetes is a plus."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t1-c",
|
|
||||||
"employer": "Acme Recruitment Group",
|
|
||||||
"title": "Senior Python Developer",
|
|
||||||
"description": (
|
|
||||||
"We are looking for a Senior Python Developer to join our backend team. "
|
|
||||||
"You will work with Fast API, PostgreSQL, and Docker in a cloud-native "
|
|
||||||
"environment. 5+ years of Python experience required. "
|
|
||||||
"Experience with AWS and Kubernetes is a plus."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
# Triple 2: Same role reposted by same agency with minor wording variations.
|
|
||||||
TRIPLE_2_SAME_AGENCY_REPOST = [
|
|
||||||
{
|
|
||||||
"id": "t2-a",
|
|
||||||
"employer": "Stockholm Tech Staffing AB",
|
|
||||||
"title": "Fullstack Engineer",
|
|
||||||
"description": (
|
|
||||||
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
|
|
||||||
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
|
|
||||||
"You will build customer-facing features and internal tools. "
|
|
||||||
"Must have experience with CI/CD pipelines."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t2-b",
|
|
||||||
"employer": "Stockholm Tech Staffing AB",
|
|
||||||
"title": "Fullstack Engineer",
|
|
||||||
"description": (
|
|
||||||
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
|
|
||||||
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
|
|
||||||
"You will build customer-facing features and internal tools. "
|
|
||||||
"Must have experience with CI/CD pipelines."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t2-c",
|
|
||||||
"employer": "Stockholm Tech Staffing",
|
|
||||||
"title": "Fullstack Engineer",
|
|
||||||
"description": (
|
|
||||||
"Fullstack Engineer wanted for a fintech startup in Stockholm. "
|
|
||||||
"Tech stack: React, TypeScript, Node.js, PostgreSQL. "
|
|
||||||
"You will build customer-facing features and internal tools. "
|
|
||||||
"Must have experience with CI/CD pipelines."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
# Triple 3: Same role, slightly different title but same description body.
|
|
||||||
# Different employers (agencies), high title + desc similarity.
|
|
||||||
TRIPLE_3_CROSS_AGENCY = [
|
|
||||||
{
|
|
||||||
"id": "t3-a",
|
|
||||||
"employer": "Data Recruiting Solutions",
|
|
||||||
"title": "Data Engineer",
|
|
||||||
"description": (
|
|
||||||
"We seek a Data Engineer to build and maintain ETL pipelines using "
|
|
||||||
"Python, Airflow, dbt, and Snowflake. You will design data models, "
|
|
||||||
"optimize queries, and ensure data quality. Experience with "
|
|
||||||
"distributed systems and Spark is required."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t3-b",
|
|
||||||
"employer": "Cloud Talent Partners",
|
|
||||||
"title": "Data Engineer",
|
|
||||||
"description": (
|
|
||||||
"We seek a Data Engineer to build and maintain ETL pipelines using "
|
|
||||||
"Python, Airflow, dbt, and Snowflake. You will design data models, "
|
|
||||||
"optimize queries, and ensure data quality. Experience with "
|
|
||||||
"distributed systems and Spark is required."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "t3-c",
|
|
||||||
"employer": "Analytics Staffing Ltd",
|
|
||||||
"title": "Data Engineer",
|
|
||||||
"description": (
|
|
||||||
"We seek a Data Engineer to build and maintain ETL pipelines using "
|
|
||||||
"Python, Airflow, dbt, and Snowflake. You will design data models, "
|
|
||||||
"optimize queries, and ensure data quality. Experience with "
|
|
||||||
"distributed systems and Spark is required."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# NEGATIVE case: legit different jobs at same agency -- must NOT cluster.
|
|
||||||
# Same agency employer but different titles and different descriptions.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY = [
|
|
||||||
{
|
|
||||||
"id": "neg-a",
|
|
||||||
"employer": "Nordic IT Consulting AB",
|
|
||||||
"title": "Frontend Developer",
|
|
||||||
"description": (
|
|
||||||
"We are looking for a Frontend Developer with expertise in React "
|
|
||||||
"and TypeScript. You will build responsive web applications and "
|
|
||||||
"work closely with our design team. Experience with CSS-in-JS and "
|
|
||||||
"accessibility standards is required."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "neg-b",
|
|
||||||
"employer": "Nordic IT Consulting AB",
|
|
||||||
"title": "DevOps Engineer",
|
|
||||||
"description": (
|
|
||||||
"We need a DevOps Engineer to manage our Kubernetes clusters and "
|
|
||||||
"CI/CD pipelines. You will work with Terraform, ArgoCD, and "
|
|
||||||
"Prometheus monitoring. Strong Linux and networking background "
|
|
||||||
"is required. AWS certification is a plus."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def triple1():
|
|
||||||
"""Agency repost triple 1: same role, two agencies + employer."""
|
|
||||||
return [dict(p) for p in TRIPLE_1_AGENCY_REPOST]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def triple2():
|
|
||||||
"""Agency repost triple 2: same agency reposts same job."""
|
|
||||||
return [dict(p) for p in TRIPLE_2_SAME_AGENCY_REPOST]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def triple3():
|
|
||||||
"""Agency repost triple 3: cross-agency same role same description."""
|
|
||||||
return [dict(p) for p in TRIPLE_3_CROSS_AGENCY]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def negative_same_agency():
|
|
||||||
"""Negative case: different jobs at same agency, must NOT cluster."""
|
|
||||||
return [dict(p) for p in NEGATIVE_DIFFERENT_JOBS_SAME_AGENCY]
|
|
||||||
|
|
@ -1,168 +0,0 @@
|
||||||
"""Tests for dedupe clustering."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from matching.dedupe import cluster
|
|
||||||
|
|
||||||
|
|
||||||
class TestClusterBasic:
|
|
||||||
def test_empty_list(self):
|
|
||||||
assert cluster([]) == {}
|
|
||||||
|
|
||||||
def test_single_posting(self):
|
|
||||||
result = cluster([
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}
|
|
||||||
])
|
|
||||||
assert len(result) == 1
|
|
||||||
assert "c1" in result
|
|
||||||
assert result["c1"] == ["a"]
|
|
||||||
|
|
||||||
def test_no_duplicates_separate_clusters(self):
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"},
|
|
||||||
{"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"},
|
|
||||||
{"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
# Each posting in its own cluster.
|
|
||||||
total_ids = sum(len(v) for v in result.values())
|
|
||||||
assert total_ids == 3
|
|
||||||
# All cluster values are singletons.
|
|
||||||
for ids in result.values():
|
|
||||||
assert len(ids) == 1
|
|
||||||
|
|
||||||
|
|
||||||
class TestClusterEmployerMatch:
|
|
||||||
def test_same_employer_clusters(self):
|
|
||||||
"""Same employer name (normalized) with similar content should cluster."""
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
|
|
||||||
{"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
# Same employer + similar title -> same cluster.
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"a", "b"}
|
|
||||||
|
|
||||||
def test_same_employer_different_content_no_cluster(self):
|
|
||||||
"""Same employer but completely different titles/descriptions should NOT cluster."""
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."},
|
|
||||||
{"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
# Same employer but different jobs -> no cluster.
|
|
||||||
assert all(len(v) == 1 for v in result.values())
|
|
||||||
|
|
||||||
def test_employer_suffix_variations_cluster(self):
|
|
||||||
"""Acme AB and Acme should cluster (normalized match)."""
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"},
|
|
||||||
{"id": "b", "employer": "Acme", "title": "Dev", "description": "y"},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"a", "b"}
|
|
||||||
|
|
||||||
|
|
||||||
class TestClusterTitleDescMatch:
|
|
||||||
def test_title_desc_high_enough(self):
|
|
||||||
"""Different employers but title >= 85 and desc >= 80 -> cluster."""
|
|
||||||
desc = (
|
|
||||||
"We are looking for a Senior Python Developer to join our backend "
|
|
||||||
"team. You will work with Fast API, PostgreSQL, and Docker."
|
|
||||||
)
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc},
|
|
||||||
{"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"a", "b"}
|
|
||||||
|
|
||||||
def test_title_high_desc_low_no_cluster(self):
|
|
||||||
"""Title similar but desc too different -> no cluster."""
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."},
|
|
||||||
{"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
# Should NOT cluster (different employers, low desc score).
|
|
||||||
assert len(result) == 2 or all(len(v) == 1 for v in result.values())
|
|
||||||
|
|
||||||
|
|
||||||
class TestClusterTransitive:
|
|
||||||
def test_transitive_clustering(self):
|
|
||||||
"""If A~B and B~C then A~C should be in same cluster."""
|
|
||||||
# A and B same employer + similar title, B and C same employer + similar title.
|
|
||||||
postings = [
|
|
||||||
{"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
|
|
||||||
{"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
|
|
||||||
{"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"a", "b", "c"}
|
|
||||||
|
|
||||||
|
|
||||||
class TestClusterSorting:
|
|
||||||
def test_cluster_ids_sorted_by_score(self):
|
|
||||||
"""Cluster with higher pairwise score should get c1."""
|
|
||||||
# Tight cluster: identical titles and descriptions (different employers).
|
|
||||||
tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes."
|
|
||||||
# Looser cluster: different employers, high title but lower desc similarity.
|
|
||||||
loose_desc_1 = "Python data engineering and pipelines with ETL tools."
|
|
||||||
loose_desc_2 = "Python data engineering and ETL work with Airflow."
|
|
||||||
postings = [
|
|
||||||
# tight cluster (different employers, high title+desc)
|
|
||||||
{"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc},
|
|
||||||
{"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc},
|
|
||||||
# loose cluster (different employers, high title but lower desc)
|
|
||||||
{"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1},
|
|
||||||
{"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2},
|
|
||||||
]
|
|
||||||
result = cluster(postings)
|
|
||||||
# Both clusters should exist.
|
|
||||||
all_ids = set()
|
|
||||||
for ids in result.values():
|
|
||||||
all_ids.update(ids)
|
|
||||||
assert all_ids == {"t1", "t2", "l1", "l2"}
|
|
||||||
# c1 should be the tight cluster (higher score: title+desc both 100).
|
|
||||||
assert set(result["c1"]) == {"t1", "t2"}
|
|
||||||
|
|
||||||
|
|
||||||
class TestAgencyRepostTriples:
|
|
||||||
"""Test the 3 agency-repost fixture triples."""
|
|
||||||
|
|
||||||
def test_triple1_all_cluster(self, triple1):
|
|
||||||
"""Triple 1: 3 postings of same role via different employers cluster."""
|
|
||||||
result = cluster(triple1)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"}
|
|
||||||
|
|
||||||
def test_triple2_all_cluster(self, triple2):
|
|
||||||
"""Triple 2: same agency reposts same job (suffix variations)."""
|
|
||||||
result = cluster(triple2)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"}
|
|
||||||
|
|
||||||
def test_triple3_all_cluster(self, triple3):
|
|
||||||
"""Triple 3: cross-agency same role same description."""
|
|
||||||
result = cluster(triple3)
|
|
||||||
assert len(result) == 1
|
|
||||||
assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"}
|
|
||||||
|
|
||||||
|
|
||||||
class TestNegativeSameAgencyDifferentJobs:
|
|
||||||
"""Negative case: different jobs at same agency must NOT cluster."""
|
|
||||||
|
|
||||||
def test_different_jobs_same_agency_no_cluster(self, negative_same_agency):
|
|
||||||
"""Different jobs at same agency must NOT cluster.
|
|
||||||
|
|
||||||
Same employer but completely different titles and descriptions.
|
|
||||||
Per the clustering rule, same employer alone is not sufficient;
|
|
||||||
some content overlap (title >= 85 OR desc >= 80) is also required.
|
|
||||||
"""
|
|
||||||
result = cluster(negative_same_agency)
|
|
||||||
all_singletons = all(len(v) == 1 for v in result.values())
|
|
||||||
assert all_singletons, "Different jobs at same agency should not cluster"
|
|
||||||
|
|
@ -1,152 +0,0 @@
|
||||||
"""Tests for keyword extraction and coverage."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from matching.keywords import extract_keywords, coverage
|
|
||||||
|
|
||||||
|
|
||||||
class TestExtractKeywords:
|
|
||||||
def test_basic_extraction(self):
|
|
||||||
text = "Python developer with Fast API experience and PostgreSQL database skills."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "python" in kws
|
|
||||||
assert "fastapi" in kws
|
|
||||||
assert "postgresql" in kws
|
|
||||||
|
|
||||||
def test_stopwords_removed(self):
|
|
||||||
text = "We are looking for a developer with experience in Python."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "we" not in kws
|
|
||||||
assert "are" not in kws
|
|
||||||
assert "for" not in kws
|
|
||||||
assert "a" not in kws
|
|
||||||
assert "in" not in kws
|
|
||||||
assert "python" in kws
|
|
||||||
assert "developer" in kws
|
|
||||||
|
|
||||||
def test_swedish_stopwords_removed(self):
|
|
||||||
text = "Vi letar efter en Python utvecklare med erfarenhet av Docker."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "vi" not in kws
|
|
||||||
assert "en" not in kws
|
|
||||||
assert "av" not in kws
|
|
||||||
assert "python" in kws
|
|
||||||
assert "docker" in kws
|
|
||||||
|
|
||||||
def test_top_n_limit(self):
|
|
||||||
text = "python python python docker docker docker kubernetes kubernetes kubernetes react react react"
|
|
||||||
kws = extract_keywords(text, top_n=2)
|
|
||||||
assert len(kws) == 2
|
|
||||||
|
|
||||||
def test_empty_text(self):
|
|
||||||
assert extract_keywords("") == []
|
|
||||||
|
|
||||||
def test_whitespace_only(self):
|
|
||||||
assert extract_keywords(" ") == []
|
|
||||||
|
|
||||||
def test_multiword_fastapi(self):
|
|
||||||
text = "Experience with fast api framework for building REST APIs."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "fastapi" in kws
|
|
||||||
|
|
||||||
def test_multiword_machine_learning(self):
|
|
||||||
text = "machine learning models for predictive analytics."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "machine-learning" in kws
|
|
||||||
|
|
||||||
def test_frequency_ordering(self):
|
|
||||||
text = "python python python docker docker kubernetes"
|
|
||||||
kws = extract_keywords(text, top_n=3)
|
|
||||||
assert kws[0] == "python"
|
|
||||||
assert kws[1] == "docker"
|
|
||||||
assert kws[2] == "kubernetes"
|
|
||||||
|
|
||||||
def test_deterministic_tie_breaking(self):
|
|
||||||
"""Ties in frequency should be broken alphabetically."""
|
|
||||||
text = "docker kubernetes"
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
# Both have frequency 1, so alphabetical: docker < kubernetes
|
|
||||||
assert kws[0] == "docker"
|
|
||||||
assert kws[1] == "kubernetes"
|
|
||||||
|
|
||||||
def test_min_token_length(self):
|
|
||||||
text = "x y z aa bb cc developer"
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "x" not in kws
|
|
||||||
assert "y" not in kws
|
|
||||||
assert "z" not in kws
|
|
||||||
assert "developer" in kws
|
|
||||||
|
|
||||||
def test_tech_terms_preserved(self):
|
|
||||||
text = "Node.js and React Native for mobile development."
|
|
||||||
kws = extract_keywords(text)
|
|
||||||
assert "nodejs" in kws
|
|
||||||
assert "react-native" in kws
|
|
||||||
|
|
||||||
|
|
||||||
class TestCoverage:
|
|
||||||
def test_full_coverage(self):
|
|
||||||
cv = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
|
|
||||||
posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes"
|
|
||||||
result = coverage(cv, posting)
|
|
||||||
assert result["ratio"] == 1.0
|
|
||||||
assert len(result["missing"]) == 0
|
|
||||||
|
|
||||||
def test_partial_coverage(self):
|
|
||||||
cv = "Python developer with PostgreSQL and Docker experience"
|
|
||||||
posting = "Python developer with Fast API PostgreSQL Docker AWS Kubernetes React"
|
|
||||||
result = coverage(cv, posting)
|
|
||||||
assert 0.0 < result["ratio"] < 1.0
|
|
||||||
assert "python" in result["matched"]
|
|
||||||
assert "postgresql" in result["matched"]
|
|
||||||
assert "docker" in result["matched"]
|
|
||||||
assert "fastapi" in result["missing"]
|
|
||||||
assert "kubernetes" in result["missing"]
|
|
||||||
|
|
||||||
def test_zero_coverage(self):
|
|
||||||
cv = "Chef with experience in French cuisine and menu planning"
|
|
||||||
posting = "Python developer with Fast API PostgreSQL Docker"
|
|
||||||
result = coverage(cv, posting)
|
|
||||||
assert result["ratio"] == 0.0
|
|
||||||
assert len(result["matched"]) == 0
|
|
||||||
assert len(result["missing"]) > 0
|
|
||||||
|
|
||||||
def test_empty_posting(self):
|
|
||||||
result = coverage("Python developer", "")
|
|
||||||
assert result == {"matched": [], "missing": [], "ratio": 0.0}
|
|
||||||
|
|
||||||
def test_empty_cv(self):
|
|
||||||
result = coverage("", "Python developer with Docker")
|
|
||||||
assert result["ratio"] == 0.0
|
|
||||||
assert len(result["matched"]) == 0
|
|
||||||
assert len(result["missing"]) > 0
|
|
||||||
|
|
||||||
def test_both_empty(self):
|
|
||||||
result = coverage("", "")
|
|
||||||
assert result == {"matched": [], "missing": [], "ratio": 0.0}
|
|
||||||
|
|
||||||
def test_ratio_calculation(self):
|
|
||||||
cv = "python docker postgresql"
|
|
||||||
posting = "python docker postgresql kubernetes"
|
|
||||||
result = coverage(cv, posting)
|
|
||||||
# 3 of 4 matched (approx, depends on stopword filtering).
|
|
||||||
assert result["ratio"] > 0.5
|
|
||||||
assert result["ratio"] <= 1.0
|
|
||||||
|
|
||||||
def test_matched_and_missing_lists(self):
|
|
||||||
cv = "python docker"
|
|
||||||
posting = "python docker kubernetes react"
|
|
||||||
result = coverage(cv, posting)
|
|
||||||
assert "python" in result["matched"]
|
|
||||||
assert "docker" in result["matched"]
|
|
||||||
assert "kubernetes" in result["missing"]
|
|
||||||
assert "react" in result["missing"]
|
|
||||||
|
|
||||||
def test_coverage_returns_dict_keys(self):
|
|
||||||
result = coverage("python", "python docker")
|
|
||||||
assert "matched" in result
|
|
||||||
assert "missing" in result
|
|
||||||
assert "ratio" in result
|
|
||||||
assert isinstance(result["matched"], list)
|
|
||||||
assert isinstance(result["missing"], list)
|
|
||||||
assert isinstance(result["ratio"], float)
|
|
||||||
|
|
@ -1,126 +0,0 @@
|
||||||
"""Tests for similarity helpers."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from matching.similarity import (
|
|
||||||
normalize_employer,
|
|
||||||
title_score,
|
|
||||||
employer_match,
|
|
||||||
desc_score,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestNormalizeEmployer:
|
|
||||||
def test_simple_lowercase(self):
|
|
||||||
assert normalize_employer("Acme") == "acme"
|
|
||||||
|
|
||||||
def test_strips_swedish_ab(self):
|
|
||||||
assert normalize_employer("Acme AB") == "acme"
|
|
||||||
|
|
||||||
def test_strips_aktiebolag(self):
|
|
||||||
assert normalize_employer("Acme Aktiebolag") == "acme"
|
|
||||||
|
|
||||||
def test_strips_consulting_suffix(self):
|
|
||||||
assert normalize_employer("Nordic IT Consulting AB") == "nordic it"
|
|
||||||
|
|
||||||
def test_strips_corp_suffix(self):
|
|
||||||
assert normalize_employer("Globex Corp.") == "globex"
|
|
||||||
|
|
||||||
def test_strips_ltd_suffix(self):
|
|
||||||
assert normalize_employer("Foo Ltd") == "foo"
|
|
||||||
|
|
||||||
def test_strips_recruitment_suffix(self):
|
|
||||||
assert normalize_employer("Acme Recruitment Group") == "acme"
|
|
||||||
|
|
||||||
def test_strips_multiple_suffixes(self):
|
|
||||||
# "Consulting AB" should strip both "AB" then "Consulting"
|
|
||||||
assert normalize_employer("Nordic Consulting AB") == "nordic"
|
|
||||||
|
|
||||||
def test_removes_punctuation(self):
|
|
||||||
assert normalize_employer("Acme, Inc.") == "acme"
|
|
||||||
|
|
||||||
def test_empty_string(self):
|
|
||||||
assert normalize_employer("") == ""
|
|
||||||
|
|
||||||
def test_whitespace_only(self):
|
|
||||||
assert normalize_employer(" ") == ""
|
|
||||||
|
|
||||||
def test_preserves_core_name_with_special_chars(self):
|
|
||||||
result = normalize_employer("Café Nu AB")
|
|
||||||
assert "café" in result or "cafe" in result
|
|
||||||
|
|
||||||
def test_dots_in_name_preserved(self):
|
|
||||||
# Punctuation (except & which gets stripped) is removed; H&M -> h m
|
|
||||||
result = normalize_employer("H&M AB")
|
|
||||||
assert result == "h m"
|
|
||||||
|
|
||||||
|
|
||||||
class TestTitleScore:
|
|
||||||
def test_identical_titles(self):
|
|
||||||
assert title_score("Senior Python Developer", "Senior Python Developer") == 100.0
|
|
||||||
|
|
||||||
def test_similar_titles_high_score(self):
|
|
||||||
score = title_score("Python Developer", "Senior Python Developer")
|
|
||||||
assert score >= 85.0
|
|
||||||
|
|
||||||
def test_different_titles_low_score(self):
|
|
||||||
score = title_score("Python Developer", "Frontend Designer")
|
|
||||||
assert score < 50.0
|
|
||||||
|
|
||||||
def test_empty_title(self):
|
|
||||||
assert title_score("", "Something") == 0.0
|
|
||||||
|
|
||||||
def test_both_empty(self):
|
|
||||||
assert title_score("", "") == 0.0
|
|
||||||
|
|
||||||
def test_order_independent(self):
|
|
||||||
# token_set_ratio is order-independent
|
|
||||||
a = "Senior Python Developer"
|
|
||||||
b = "Developer Python Senior"
|
|
||||||
assert title_score(a, b) == 100.0
|
|
||||||
|
|
||||||
|
|
||||||
class TestEmployerMatch:
|
|
||||||
def test_same_name_matches(self):
|
|
||||||
assert employer_match("Acme AB", "Acme AB") is True
|
|
||||||
|
|
||||||
def test_suffix_variation_matches(self):
|
|
||||||
assert employer_match("Acme AB", "Acme") is True
|
|
||||||
|
|
||||||
def test_different_employers_no_match(self):
|
|
||||||
assert employer_match("Acme AB", "Globex AB") is False
|
|
||||||
|
|
||||||
def test_consulting_variations_match(self):
|
|
||||||
assert employer_match("Nordic IT Consulting AB", "Nordic IT") is True
|
|
||||||
|
|
||||||
def test_empty_no_match(self):
|
|
||||||
assert employer_match("", "") is False
|
|
||||||
|
|
||||||
def test_one_empty_no_match(self):
|
|
||||||
assert employer_match("Acme", "") is False
|
|
||||||
|
|
||||||
|
|
||||||
class TestDescScore:
|
|
||||||
def test_identical_descriptions(self):
|
|
||||||
desc = "We are looking for a Python developer with 5 years experience."
|
|
||||||
assert desc_score(desc, desc) == 100.0
|
|
||||||
|
|
||||||
def test_similar_descriptions_high(self):
|
|
||||||
a = "We are looking for a Python developer with 5 years experience."
|
|
||||||
b = "We are looking for a Python developer with 5 years experience in web."
|
|
||||||
assert desc_score(a, b) >= 80.0
|
|
||||||
|
|
||||||
def test_different_descriptions_low(self):
|
|
||||||
a = "We need a frontend developer skilled in React and CSS."
|
|
||||||
b = "Looking for a data scientist with Python and SQL expertise."
|
|
||||||
assert desc_score(a, b) < 50.0
|
|
||||||
|
|
||||||
def test_empty_desc(self):
|
|
||||||
assert desc_score("", "something") == 0.0
|
|
||||||
|
|
||||||
def test_truncation(self):
|
|
||||||
# Test that truncation to max_chars works.
|
|
||||||
long_a = "Python " * 1000
|
|
||||||
long_b = "Python " * 1000
|
|
||||||
score = desc_score(long_a, long_b, max_chars=100)
|
|
||||||
assert score == 100.0
|
|
||||||
Loading…
Reference in a new issue