jobhunt-platform/packages/matching/tests/test_dedupe.py

"""Tests for dedupe clustering."""

from __future__ import annotations

from matching.dedupe import cluster


class TestClusterBasic:
    def test_empty_list(self):
        assert cluster([]) == {}

    def test_single_posting(self):
        result = cluster([
            {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"}
        ])
        assert len(result) == 1
        assert "c1" in result
        assert result["c1"] == ["a"]

    def test_no_duplicates_separate_clusters(self):
        postings = [
            {"id": "a", "employer": "Acme AB", "title": "Python Dev", "description": "Python backend"},
            {"id": "b", "employer": "Globex AB", "title": "React Dev", "description": "React frontend"},
            {"id": "c", "employer": "Foo Ltd", "title": "Data Scientist", "description": "ML pipelines"},
        ]
        result = cluster(postings)
        # Each posting in its own cluster.
        total_ids = sum(len(v) for v in result.values())
        assert total_ids == 3
        # All cluster values are singletons.
        for ids in result.values():
            assert len(ids) == 1


class TestClusterEmployerMatch:
    def test_same_employer_clusters(self):
        """Same employer name (normalized) with similar content should cluster."""
        postings = [
            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
            {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
        ]
        result = cluster(postings)
        # Same employer + similar title -> same cluster.
        assert len(result) == 1
        assert set(result["c1"]) == {"a", "b"}

    def test_same_employer_different_content_no_cluster(self):
        """Same employer but completely different titles/descriptions should NOT cluster."""
        postings = [
            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "We need a Python developer for backend work."},
            {"id": "b", "employer": "Acme AB", "title": "Chef", "description": "Looking for a head chef for our restaurant kitchen."},
        ]
        result = cluster(postings)
        # Same employer but different jobs -> no cluster.
        assert all(len(v) == 1 for v in result.values())

    def test_employer_suffix_variations_cluster(self):
        """Acme AB and Acme should cluster (normalized match)."""
        postings = [
            {"id": "a", "employer": "Acme AB", "title": "Dev", "description": "x"},
            {"id": "b", "employer": "Acme", "title": "Dev", "description": "y"},
        ]
        result = cluster(postings)
        assert len(result) == 1
        assert set(result["c1"]) == {"a", "b"}


class TestClusterTitleDescMatch:
    def test_title_desc_high_enough(self):
        """Different employers but title >= 85 and desc >= 80 -> cluster."""
        desc = (
            "We are looking for a Senior Python Developer to join our backend "
            "team. You will work with Fast API, PostgreSQL, and Docker."
        )
        postings = [
            {"id": "a", "employer": "Agency One AB", "title": "Senior Python Developer", "description": desc},
            {"id": "b", "employer": "Agency Two AB", "title": "Senior Python Developer", "description": desc},
        ]
        result = cluster(postings)
        assert len(result) == 1
        assert set(result["c1"]) == {"a", "b"}

    def test_title_high_desc_low_no_cluster(self):
        """Title similar but desc too different -> no cluster."""
        postings = [
            {"id": "a", "employer": "Agency A", "title": "Python Developer", "description": "We need a Python developer for backend work with Django."},
            {"id": "b", "employer": "Agency B", "title": "Python Developer", "description": "Looking for someone to teach Python to high school students."},
        ]
        result = cluster(postings)
        # Should NOT cluster (different employers, low desc score).
        assert len(result) == 2 or all(len(v) == 1 for v in result.values())


class TestClusterTransitive:
    def test_transitive_clustering(self):
        """If A~B and B~C then A~C should be in same cluster."""
        # A and B same employer + similar title, B and C same employer + similar title.
        postings = [
            {"id": "a", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development"},
            {"id": "b", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development senior"},
            {"id": "c", "employer": "Acme AB", "title": "Python Developer", "description": "Python backend development lead"},
        ]
        result = cluster(postings)
        assert len(result) == 1
        assert set(result["c1"]) == {"a", "b", "c"}


class TestClusterSorting:
    def test_cluster_ids_sorted_by_score(self):
        """Cluster with higher pairwise score should get c1."""
        # Tight cluster: identical titles and descriptions (different employers).
        tight_desc = "Python backend developer with Fast API and PostgreSQL and Docker and AWS and Kubernetes."
        # Looser cluster: different employers, high title but lower desc similarity.
        loose_desc_1 = "Python data engineering and pipelines with ETL tools."
        loose_desc_2 = "Python data engineering and ETL work with Airflow."
        postings = [
            # tight cluster (different employers, high title+desc)
            {"id": "t1", "employer": "Agency A", "title": "Python Developer", "description": tight_desc},
            {"id": "t2", "employer": "Agency B", "title": "Python Developer", "description": tight_desc},
            # loose cluster (different employers, high title but lower desc)
            {"id": "l1", "employer": "Agency C", "title": "Python Developer", "description": loose_desc_1},
            {"id": "l2", "employer": "Agency D", "title": "Python Developer", "description": loose_desc_2},
        ]
        result = cluster(postings)
        # Both clusters should exist.
        all_ids = set()
        for ids in result.values():
            all_ids.update(ids)
        assert all_ids == {"t1", "t2", "l1", "l2"}
        # c1 should be the tight cluster (higher score: title+desc both 100).
        assert set(result["c1"]) == {"t1", "t2"}


class TestAgencyRepostTriples:
    """Test the 3 agency-repost fixture triples."""

    def test_triple1_all_cluster(self, triple1):
        """Triple 1: 3 postings of same role via different employers cluster."""
        result = cluster(triple1)
        assert len(result) == 1
        assert set(result["c1"]) == {"t1-a", "t1-b", "t1-c"}

    def test_triple2_all_cluster(self, triple2):
        """Triple 2: same agency reposts same job (suffix variations)."""
        result = cluster(triple2)
        assert len(result) == 1
        assert set(result["c1"]) == {"t2-a", "t2-b", "t2-c"}

    def test_triple3_all_cluster(self, triple3):
        """Triple 3: cross-agency same role same description."""
        result = cluster(triple3)
        assert len(result) == 1
        assert set(result["c1"]) == {"t3-a", "t3-b", "t3-c"}


class TestNegativeSameAgencyDifferentJobs:
    """Negative case: different jobs at same agency must NOT cluster."""

    def test_different_jobs_same_agency_no_cluster(self, negative_same_agency):
        """Different jobs at same agency must NOT cluster.

        Same employer but completely different titles and descriptions.
        Per the clustering rule, same employer alone is not sufficient;
        some content overlap (title >= 85 OR desc >= 80) is also required.
        """
        result = cluster(negative_same_agency)
        all_singletons = all(len(v) == 1 for v in result.values())
        assert all_singletons, "Different jobs at same agency should not cluster"