0e13ee5d51
- Connector protocol with fetch(query: SearchQuery) -> list[RawPosting] - ArbetsformedlingenConnector: official Platsbanken API, field mapping (headline->title, employer.name->company, webpage_url->url, description.text->description, id->external_id), region mapping, polite User-Agent - GenericUrlConnector: readability extraction (prefers main/article, strips nav/footer/script/style), title/company guess, Cloudflare challenge detection -> UnsupportedSite - dedupe helper: (source, url) + external_id keying - normalizer: RawPosting -> JobPosting (job_posting table shape) - 37 tests, all passing, recorded fixtures (no live network) - README with usage and mock examples
87 lines
No EOL
2.8 KiB
Python
87 lines
No EOL
2.8 KiB
Python
"""Tests for the normalizer and connector protocol."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from connectors import (
|
|
Connector,
|
|
GenericUrlConnector,
|
|
JobPosting,
|
|
RawPosting,
|
|
normalize,
|
|
)
|
|
from connectors.arbetsformedlingen import ArbetsformedlingenConnector
|
|
|
|
|
|
class TestNormalize:
|
|
"""Test the normalize function."""
|
|
|
|
def test_basic_normalization(self):
|
|
"""Test that normalize maps all fields correctly."""
|
|
raw = RawPosting(
|
|
source="arbetsformedlingen",
|
|
external_id="12345",
|
|
url="https://example.com/job/12345",
|
|
company="Test Corp",
|
|
title="Test Job",
|
|
location="Malmo",
|
|
description="A test job description.",
|
|
raw={"id": "12345", "extra": "data"},
|
|
)
|
|
job = normalize(raw)
|
|
|
|
assert job.source == "arbetsformedlingen"
|
|
assert job.external_id == "12345"
|
|
assert job.url == "https://example.com/job/12345"
|
|
assert job.company == "Test Corp"
|
|
assert job.title == "Test Job"
|
|
assert job.location == "Malmo"
|
|
assert job.description == "A test job description."
|
|
assert job.raw == {"id": "12345", "extra": "data"}
|
|
|
|
def test_none_fields_preserved(self):
|
|
"""Test that None fields are preserved."""
|
|
raw = RawPosting(
|
|
source="generic_url",
|
|
external_id=None,
|
|
url="https://example.com/page",
|
|
company="",
|
|
title="Unknown",
|
|
)
|
|
job = normalize(raw)
|
|
|
|
assert job.external_id is None
|
|
assert job.location is None
|
|
assert job.description == ""
|
|
assert job.raw == {}
|
|
|
|
def test_swedish_chars_preserved(self):
|
|
"""Test that Swedish characters are preserved through normalization."""
|
|
raw = RawPosting(
|
|
source="arbetsformedlingen",
|
|
external_id="1",
|
|
url="https://example.com/1",
|
|
company="Nordic Solutions AB",
|
|
title="Utvecklare",
|
|
location="Lund",
|
|
description="Vi soker en utvecklare.",
|
|
)
|
|
job = normalize(raw)
|
|
|
|
assert "Solutions" in job.company
|
|
assert "Utvecklare" in job.title
|
|
assert "soker" in job.description
|
|
assert "Lund" in (job.location or "")
|
|
|
|
|
|
class TestConnectorProtocol:
|
|
"""Test that connectors satisfy the Connector protocol."""
|
|
|
|
def test_af_connector_is_connector(self):
|
|
"""Test that ArbetsformedlingenConnector satisfies the Connector protocol."""
|
|
connector = ArbetsformedlingenConnector()
|
|
assert isinstance(connector, Connector)
|
|
|
|
def test_generic_url_connector_is_connector(self):
|
|
"""Test that GenericUrlConnector satisfies the Connector protocol."""
|
|
connector = GenericUrlConnector()
|
|
assert isinstance(connector, Connector) |