jobhunt-platform/packages/connectors/tests/test_generic_url.py

"""Tests for the GenericUrlConnector.

Uses recorded HTML fixtures (no live network). Fixtures are loaded into
an httpx.MockTransport.
"""

from __future__ import annotations

from pathlib import Path

import httpx
import pytest

from connectors.exceptions import UnsupportedSite
from connectors.generic_url import GenericUrlConnector
from connectors.models import SearchQuery

FIXTURES_DIR = Path(__file__).parent / "fixtures"


def _load_fixture(name: str) -> str:
    """Load a fixture file as raw text."""
    return (FIXTURES_DIR / name).read_text(encoding="utf-8")


def _make_client(
    response_body: str,
    status_code: int = 200,
    headers: dict | None = None,
) -> httpx.Client:
    """Create an httpx.Client with a mock transport returning the fixture."""
    default_headers = {"content-type": "text/html; charset=utf-8"}
    if headers:
        default_headers.update(headers)

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(status_code, content=response_body, headers=default_headers)

    transport = httpx.MockTransport(handler)
    return httpx.Client(transport=transport)


class TestGenericUrlExtraction:
    """Test readability extraction from a messy HTML page."""

    def test_extracts_title_from_h1(self):
        """Test that the h1 tag is used as the title."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        assert len(results) == 1
        posting = results[0]
        assert posting.title == "Senior Backend Engineer"

    def test_extracts_company_from_meta(self):
        """Test that og:site_name meta tag is used as company name."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        assert results[0].company == "Acme Corp"

    def test_strips_nav_and_footer(self):
        """Test that nav and footer content is removed from description."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        description = results[0].description
        # Nav links should not appear
        assert "Home" not in description or "Requirements" in description
        assert "Privacy Policy" not in description
        assert "Terms of Service" not in description
        assert "Login | Register" not in description
        # Footer copyright should not appear
        assert "2026 JobBoard" not in description
        # Job content should be present
        assert "Python" in description
        assert "FastAPI" in description
        assert "PostgreSQL" in description

    def test_strips_script_and_style(self):
        """Test that script and style tags are removed."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        description = results[0].description
        assert "analytics" not in description.lower()
        assert "var " not in description
        assert "function()" not in description

    def test_source_is_generic_url(self):
        """Test that source is set to 'generic_url'."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        assert results[0].source == "generic_url"

    def test_url_preserved(self):
        """Test that the original URL is preserved in the result."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        url = "https://example.com/job/123"
        results = connector.fetch(SearchQuery(query=url))

        assert results[0].url == url

    def test_external_id_is_none(self):
        """Test that external_id is None for generic URL postings."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        assert results[0].external_id is None

    def test_description_is_clean(self):
        """Test that description text is clean and readable."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        description = results[0].description
        # Should not have excessive blank lines
        assert "\n\n\n" not in description
        # Should start with the job title or job content
        assert len(description) > 50

    def test_location_extracted_from_text(self):
        """Test that location is extracted from description text."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        # The fixture contains "Location: Malmo, Sweden"
        assert results[0].location is not None
        assert "Malmo" in results[0].location

    def test_raw_contains_metadata(self):
        """Test that the raw field contains response metadata."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)
        results = connector.fetch(SearchQuery(query="https://example.com/job/123"))

        raw = results[0].raw
        assert raw["status_code"] == 200
        assert raw["content_length"] > 0


class TestCloudflareDetection:
    """Test Cloudflare challenge page detection."""

    def test_raises_on_cloudflare_challenge(self):
        """Test that UnsupportedSite is raised for Cloudflare challenge pages."""
        html = _load_fixture("cloudflare_challenge.html")
        client = _make_client(
            html,
            headers={
                "server": "cloudflare",
                "cf-ray": "abc123-xyz789",
            },
        )
        connector = GenericUrlConnector(client=client)

        with pytest.raises(UnsupportedSite) as exc_info:
            connector.fetch(SearchQuery(query="https://protected-site.com/job/1"))

        assert "protected-site.com" in str(exc_info.value)
        assert "Cloudflare" in str(exc_info.value)

    def test_raises_on_challenge_without_server_header(self):
        """Test detection of challenge pages without cloudflare server header."""
        # A page with cf-browser-verification but no CF server header
        html = '<html><body><div id="cf-browser-verification">Loading...</div></body></html>'
        client = _make_client(html)
        connector = GenericUrlConnector(client=client)

        with pytest.raises(UnsupportedSite):
            connector.fetch(SearchQuery(query="https://sneaky-site.com/job/1"))

    def test_normal_cf_proxied_site_is_ok(self):
        """Test that a normal site behind CF (with content) is NOT flagged."""
        html = _load_fixture("generic_job_page.html")
        client = _make_client(
            html,
            headers={
                "server": "cloudflare",
                "cf-ray": "abc123-xyz789",
            },
        )
        connector = GenericUrlConnector(client=client)

        # Should NOT raise -- this is a normal page that happens to be behind CF
        results = connector.fetch(SearchQuery(query="https://cf-proxied-site.com/job/1"))
        assert len(results) == 1
        assert results[0].title == "Senior Backend Engineer"

    def test_unsupported_site_exception_has_url(self):
        """Test that UnsupportedSite exception carries the URL."""
        html = _load_fixture("cloudflare_challenge.html")
        client = _make_client(
            html,
            headers={"server": "cloudflare", "cf-ray": "xyz"},
        )
        connector = GenericUrlConnector(client=client)

        with pytest.raises(UnsupportedSite) as exc_info:
            connector.fetch(SearchQuery(query="https://example.org/protected"))

        assert exc_info.value.url == "https://example.org/protected"