- Connector protocol with fetch(query: SearchQuery) -> list[RawPosting] - ArbetsformedlingenConnector: official Platsbanken API, field mapping (headline->title, employer.name->company, webpage_url->url, description.text->description, id->external_id), region mapping, polite User-Agent - GenericUrlConnector: readability extraction (prefers main/article, strips nav/footer/script/style), title/company guess, Cloudflare challenge detection -> UnsupportedSite - dedupe helper: (source, url) + external_id keying - normalizer: RawPosting -> JobPosting (job_posting table shape) - 37 tests, all passing, recorded fixtures (no live network) - README with usage and mock examples
191 lines
No EOL
7.3 KiB
Python
191 lines
No EOL
7.3 KiB
Python
"""Tests for the Arbetsformedlingen connector.
|
|
|
|
Uses a recorded JSON fixture (no live network). The fixture is loaded
|
|
into an httpx.MockTransport that returns the recorded response.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from connectors.arbetsformedlingen import ArbetsformedlingenConnector
|
|
from connectors.models import SearchQuery
|
|
|
|
FIXTURES_DIR = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def _load_fixture(name: str) -> str:
|
|
"""Load a fixture file as raw text."""
|
|
return (FIXTURES_DIR / name).read_text(encoding="utf-8")
|
|
|
|
|
|
def _make_client(response_body: str, status_code: int = 200) -> httpx.Client:
|
|
"""Create an httpx.Client with a mock transport returning the fixture."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(status_code, content=response_body, headers={
|
|
"content-type": "application/json",
|
|
})
|
|
transport = httpx.MockTransport(handler)
|
|
return httpx.Client(transport=transport)
|
|
|
|
|
|
class TestArbetsformedlingenMapping:
|
|
"""Test field mapping from AF API response to RawPosting."""
|
|
|
|
def test_basic_mapping(self):
|
|
"""Test that API fields map correctly to RawPosting fields."""
|
|
fixture = _load_fixture("af_search_response.json")
|
|
client = _make_client(fixture)
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="python", limit=3))
|
|
|
|
assert len(results) == 3
|
|
|
|
# First hit
|
|
first = results[0]
|
|
assert first.source == "arbetsformedlingen"
|
|
assert first.external_id == "12345678901"
|
|
assert first.title == "Senior Python Developer"
|
|
assert first.company == "Tech Innovators AB"
|
|
assert first.url == "https://arbetsformedlingen.se/platsbanken/annonser/12345678901"
|
|
assert "FastAPI" in first.description
|
|
assert first.location == "Malmo"
|
|
# raw should contain the original hit
|
|
assert first.raw["id"] == "12345678901"
|
|
|
|
def test_swedish_text_preserved(self):
|
|
"""Test that Swedish characters (a, a, o) are preserved."""
|
|
fixture = _load_fixture("af_search_response.json")
|
|
client = _make_client(fixture)
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="utvecklare", limit=3))
|
|
|
|
second = results[1]
|
|
assert second.title == "Fullstack Utvecklare"
|
|
assert "Nordic Solutions AB" in second.company
|
|
assert "soker" in second.description
|
|
assert "miljo" in second.description
|
|
|
|
def test_missing_description_field(self):
|
|
"""Test graceful handling when description is missing."""
|
|
fixture_data = {
|
|
"hits": [
|
|
{
|
|
"id": "999",
|
|
"headline": "No Description Job",
|
|
"employer": {"name": "Empty Corp"},
|
|
"webpage_url": "https://example.com/job/999",
|
|
}
|
|
]
|
|
}
|
|
client = _make_client(json.dumps(fixture_data))
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="test", limit=1))
|
|
|
|
assert len(results) == 1
|
|
assert results[0].description == ""
|
|
assert results[0].title == "No Description Job"
|
|
assert results[0].company == "Empty Corp"
|
|
|
|
def test_missing_employer_name(self):
|
|
"""Test graceful handling when employer name is missing."""
|
|
fixture_data = {
|
|
"hits": [
|
|
{
|
|
"id": "888",
|
|
"headline": "Mystery Job",
|
|
"webpage_url": "https://example.com/job/888",
|
|
"description": {"text": "A job with no employer name."},
|
|
}
|
|
]
|
|
}
|
|
client = _make_client(json.dumps(fixture_data))
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="test", limit=1))
|
|
|
|
assert len(results) == 1
|
|
assert results[0].company == ""
|
|
|
|
def test_empty_hits(self):
|
|
"""Test handling of an empty results list."""
|
|
fixture_data = {"hits": [], "total": {"value": 0, "relation": "eq"}}
|
|
client = _make_client(json.dumps(fixture_data))
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="nonexistent", limit=10))
|
|
|
|
assert results == []
|
|
|
|
def test_region_param_passed(self):
|
|
"""Test that region parameter is mapped and sent to the API."""
|
|
fixture = _load_fixture("af_search_response.json")
|
|
captured_params: dict = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured_params.update(dict(request.url.params))
|
|
return httpx.Response(200, content=fixture, headers={
|
|
"content-type": "application/json",
|
|
})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
client = httpx.Client(transport=transport)
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
connector.fetch(SearchQuery(query="dev", region="malmo"))
|
|
|
|
assert captured_params.get("q") == "dev"
|
|
assert captured_params.get("region") == "Skane lan"
|
|
|
|
def test_limit_param_passed(self):
|
|
"""Test that limit parameter is sent to the API."""
|
|
fixture = _load_fixture("af_search_response.json")
|
|
captured_params: dict = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured_params.update(dict(request.url.params))
|
|
return httpx.Response(200, content=fixture, headers={
|
|
"content-type": "application/json",
|
|
})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
client = httpx.Client(transport=transport)
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
connector.fetch(SearchQuery(query="dev", limit=50))
|
|
|
|
assert captured_params.get("limit") == "50"
|
|
|
|
def test_workplace_address_municipality_fallback(self):
|
|
"""Test that municipality is used when city is missing."""
|
|
fixture_data = {
|
|
"hits": [
|
|
{
|
|
"id": "777",
|
|
"headline": "Rural Job",
|
|
"employer": {"name": "Rural Corp"},
|
|
"webpage_url": "https://example.com/job/777",
|
|
"description": {"text": "Work in the countryside."},
|
|
"workplace_address": {
|
|
"municipality": "Helsingborg",
|
|
"country": "Sverige",
|
|
},
|
|
}
|
|
]
|
|
}
|
|
client = _make_client(json.dumps(fixture_data))
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="test", limit=1))
|
|
|
|
assert results[0].location == "Helsingborg"
|
|
|
|
def test_raw_preserves_original_hit(self):
|
|
"""Test that the raw field preserves the full original hit data."""
|
|
fixture = _load_fixture("af_search_response.json")
|
|
client = _make_client(fixture)
|
|
connector = ArbetsformedlingenConnector(client=client)
|
|
results = connector.fetch(SearchQuery(query="dev", limit=3))
|
|
|
|
third = results[2]
|
|
assert third.raw["occupation"]["label"] == "DevOps-ingenjor"
|
|
assert third.raw["publication_date"] == "2026-07-30T09:00:00+02:00" |