0e13ee5d51
- Connector protocol with fetch(query: SearchQuery) -> list[RawPosting] - ArbetsformedlingenConnector: official Platsbanken API, field mapping (headline->title, employer.name->company, webpage_url->url, description.text->description, id->external_id), region mapping, polite User-Agent - GenericUrlConnector: readability extraction (prefers main/article, strips nav/footer/script/style), title/company guess, Cloudflare challenge detection -> UnsupportedSite - dedupe helper: (source, url) + external_id keying - normalizer: RawPosting -> JobPosting (job_posting table shape) - 37 tests, all passing, recorded fixtures (no live network) - README with usage and mock examples
28 lines
No EOL
584 B
TOML
28 lines
No EOL
584 B
TOML
[project]
|
|
name = "connectors"
|
|
version = "0.1.0"
|
|
description = "Job source connectors: Arbetsformedlingen API, generic URL extractor, dedupe, normalizer."
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"beautifulsoup4>=4.12",
|
|
"lxml>=5.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=8.0",
|
|
"pytest-asyncio>=0.24",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/connectors"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
pythonpath = ["src"]
|
|
asyncio_mode = "auto" |