"""Tests for dedupe clustering.""" from __future__ import annotations from matching.dedupe import cluster class TestClusterBasic: def test_empty_list(self): assert cluster([]) == {} def test_single_posting(self): result = cluster([ {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"} ]) assert len(result) == 1 assert "c1" in result assert result["c1"] == ["a"] def test_no_duplicates_separate_clusters(self): postings = [ {"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"}, {"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"}, {"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"}, ] result = cluster(postings) # Each posting in its own cluster. total_ids = sum(len(v) for v in result.values()) assert total_ids == 3 # All cluster values are singletons. for ids in result.values(): assert len(ids) == 1 class TestClusterEmployerMatch: def test_same_employer_clusters(self): """Same employer name (normalized) with similar content should cluster.""" postings = [ {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, ] result = cluster(postings) # Same employer + similar title -> same cluster. assert len(result) == 1 assert set(result["c1"]) == {"a", "b"} def test_same_employer_different_content_no_cluster(self): """Same employer but completely different titles/descriptions should NOT cluster.""" postings = [ {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."}, {"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."}, ] result = cluster(postings) # Same employer but different jobs -> no cluster. assert all(len(v) == 1 for v in result.values()) def test_employer_suffix_variations_cluster(self): """Acme AB and Acme should cluster (normalized match).""" postings = [ {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}, {"id": "b", "employer": "Acme", "title": "Dev", "description": "y"}, ] result = cluster(postings) assert len(result) == 1 assert set(result["c1"]) == {"a", "b"} class TestClusterTitleDescMatch: def test_title_desc_high_enough(self): """Different employers but title >= 85 and desc >= 80 -> cluster.""" desc = ( "We are looking for a Senior Python Developer to join our backend " "team. You will work with Fast API, PostgreSQL, and Docker." ) postings = [ {"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc}, {"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc}, ] result = cluster(postings) assert len(result) == 1 assert set(result["c1"]) == {"a", "b"} def test_title_high_desc_low_no_cluster(self): """Title similar but desc too different -> no cluster.""" postings = [ {"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."}, {"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."}, ] result = cluster(postings) # Should NOT cluster (different employers, low desc score). assert len(result) == 2 or all(len(v) == 1 for v in result.values()) class TestClusterTransitive: def test_transitive_clustering(self): """If A~B and B~C then A~C should be in same cluster.""" # A and B same employer + similar title, B and C same employer + similar title. postings = [ {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"}, {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"}, {"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"}, ] result = cluster(postings) assert len(result) == 1 assert set(result["c1"]) == {"a", "b", "c"} class TestClusterSorting: def test_cluster_ids_sorted_by_score(self): """Cluster with higher pairwise score should get c1.""" # Tight cluster: identical titles and descriptions (different employers). tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes." # Looser cluster: different employers, high title but lower desc similarity. loose_desc_1 = "Python data engineering and pipelines with ETL tools." loose_desc_2 = "Python data engineering and ETL work with Airflow." postings = [ # tight cluster (different employers, high title+desc) {"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc}, {"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc}, # loose cluster (different employers, high title but lower desc) {"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1}, {"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2}, ] result = cluster(postings) # Both clusters should exist. all_ids = set() for ids in result.values(): all_ids.update(ids) assert all_ids == {"t1", "t2", "l1", "l2"} # c1 should be the tight cluster (higher score: title+desc both 100). assert set(result["c1"]) == {"t1", "t2"} class TestAgencyRepostTriples: """Test the 3 agency-repost fixture triples.""" def test_triple1_all_cluster(self, triple1): """Triple 1: 3 postings of same role via different employers cluster.""" result = cluster(triple1) assert len(result) == 1 assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"} def test_triple2_all_cluster(self, triple2): """Triple 2: same agency reposts same job (suffix variations).""" result = cluster(triple2) assert len(result) == 1 assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"} def test_triple3_all_cluster(self, triple3): """Triple 3: cross-agency same role same description.""" result = cluster(triple3) assert len(result) == 1 assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"} class TestNegativeSameAgencyDifferentJobs: """Negative case: different jobs at same agency must NOT cluster.""" def test_different_jobs_same_agency_no_cluster(self, negative_same_agency): """Different jobs at same agency must NOT cluster. Same employer but completely different titles and descriptions. Per the clustering rule, same employer alone is not sufficient; some content overlap (title >= 85 OR desc >= 80) is also required. """ result = cluster(negative_same_agency) all_singletons = all(len(v) == 1 for v in result.values()) assert all_singletons, "Different jobs at same agency should not cluster"