From 8dceedca2ba148a5d1312481d4b0128bec83e3f2 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 20:45:11 +0000 Subject: [PATCH] WA2: llm-gateway deterministic mocks for email_classify (CHEAP), cv_tailor (STRONG), deadline_extract (CHEAP) + 9 tests --- .../llm-gateway/src/llm_gateway/config.py | 6 + packages/llm-gateway/src/llm_gateway/mock.py | 35 +++++ packages/llm-gateway/tests/test_gateway.py | 3 + packages/llm-gateway/tests/test_mock_v11.py | 125 ++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 packages/llm-gateway/tests/test_mock_v11.py diff --git a/packages/llm-gateway/src/llm_gateway/config.py b/packages/llm-gateway/src/llm_gateway/config.py index f12757e..def6ee9 100644 --- a/packages/llm-gateway/src/llm_gateway/config.py +++ b/packages/llm-gateway/src/llm_gateway/config.py @@ -30,9 +30,12 @@ TASK_CLASS_MAP: dict[str, TaskClass] = { "score": TaskClass.CHEAP, "extract": TaskClass.CHEAP, "cv_assist": TaskClass.CHEAP, + "email_classify": TaskClass.CHEAP, + "deadline_extract": TaskClass.CHEAP, "cl_critique": TaskClass.STRONG, "critique": TaskClass.STRONG, "research": TaskClass.STRONG, + "cv_tailor": TaskClass.STRONG, } # Default budgets (max output tokens) per task name. @@ -40,9 +43,12 @@ DEFAULT_BUDGETS: dict[str, int] = { "score": 2000, "extract": 4000, "cv_assist": 2000, + "email_classify": 1000, + "deadline_extract": 500, "cl_critique": 4000, "critique": 6000, "research": 4000, + "cv_tailor": 6000, } diff --git a/packages/llm-gateway/src/llm_gateway/mock.py b/packages/llm-gateway/src/llm_gateway/mock.py index 598bfa8..f14fd9a 100644 --- a/packages/llm-gateway/src/llm_gateway/mock.py +++ b/packages/llm-gateway/src/llm_gateway/mock.py @@ -59,6 +59,41 @@ MOCK_OUTPUTS: dict[str, dict] = { "summary": "The company is a mid-size tech firm focused on cloud infrastructure.", "key_points": ["Founded in 2015", "Series B funding", "Remote-first culture"], }, + "email_classify": { + "classification": "interview_invite", + "state_proposal": "interviewing", + "reason": "The email contains an invitation to schedule an interview.", + }, + "cv_tailor": { + "tailored_cv": { + "summary": "Senior Python Developer with 6+ years building scalable backend systems.", + "skills": [ + "Python", + "Fast API", + "PostgreSQL", + "Docker", + "Kubernetes", + "AWS", + ], + "experience": [ + { + "company": "TechCorp", + "role": "Senior Backend Engineer", + "bullets": [ + "Led migration of monolith to microservices using Fast API", + "Reduced API latency by 40% through query optimization and caching", + ], + }, + ], + }, + "change_log": [ + {"action": "reordered", "detail": "Moved Python and Fast API to top of skills"}, + {"action": "rephrased", "detail": "Rewrote first experience bullet to emphasize Fast API"}, + ], + }, + "deadline_extract": { + "apply_by": None, + }, } # Default mock output for unknown task names. diff --git a/packages/llm-gateway/tests/test_gateway.py b/packages/llm-gateway/tests/test_gateway.py index 04786ad..4ef2442 100644 --- a/packages/llm-gateway/tests/test_gateway.py +++ b/packages/llm-gateway/tests/test_gateway.py @@ -270,8 +270,11 @@ class TestGatewayConfig: assert config.get_task_class("score") == TaskClass.CHEAP assert config.get_task_class("extract") == TaskClass.CHEAP assert config.get_task_class("cv_assist") == TaskClass.CHEAP + assert config.get_task_class("email_classify") == TaskClass.CHEAP + assert config.get_task_class("deadline_extract") == TaskClass.CHEAP assert config.get_task_class("critique") == TaskClass.STRONG assert config.get_task_class("cl_critique") == TaskClass.STRONG + assert config.get_task_class("cv_tailor") == TaskClass.STRONG def test_get_model_routing(self) -> None: config = mock_config(cheap_model="cheap-model", strong_model="strong-model") diff --git a/packages/llm-gateway/tests/test_mock_v11.py b/packages/llm-gateway/tests/test_mock_v11.py new file mode 100644 index 0000000..1760d37 --- /dev/null +++ b/packages/llm-gateway/tests/test_mock_v11.py @@ -0,0 +1,125 @@ +"""Tests for new v1.1 mock tasks: email_classify, cv_tailor, deadline_extract.""" + +from __future__ import annotations + +import pytest + +from llm_gateway.config import GatewayConfig, ProviderConfig, TaskClass +from llm_gateway.gateway import Gateway +from llm_gateway.mock import get_mock_output, MOCK_OUTPUTS + + +def mock_config(**overrides) -> GatewayConfig: + """Build a config in mock mode (no API key).""" + primary = ProviderConfig( + name="primary", + base_url="https://mock.example.com/v1", + api_key="", + model="glm-5.2", + ) + defaults = { + "primary": primary, + "fallback": None, + "cheap_model": "glm-5.2", + "strong_model": "glm-5.2", + "budgets": { + "score": 2000, + "extract": 4000, + "email_classify": 1000, + "deadline_extract": 500, + "cv_tailor": 6000, + "default": 4000, + }, + "max_retries": 2, + } + defaults.update(overrides) + return GatewayConfig(**defaults) + + +class TestEmailClassifyMock: + async def test_email_classify_returns_deterministic(self) -> None: + """email_classify mock returns interview_invite classification.""" + config = mock_config() + gw = Gateway(config) + result_a = await gw.run_task("email_classify", "Email from recruiter") + result_b = await gw.run_task("email_classify", "Email from recruiter") + assert result_a == result_b + assert result_a["classification"] == "interview_invite" + assert result_a["state_proposal"] == "interviewing" + assert "reason" in result_a + await gw.aclose() + + async def test_email_classify_is_cheap(self) -> None: + """email_classify should be classified as CHEAP.""" + config = mock_config() + assert config.get_task_class("email_classify") == TaskClass.CHEAP + assert config.get_model("email_classify") == config.cheap_model + + def test_email_classify_in_mock_outputs(self) -> None: + """email_classify should be in MOCK_OUTPUTS.""" + assert "email_classify" in MOCK_OUTPUTS + output = get_mock_output("email_classify") + assert output["classification"] == "interview_invite" + assert output["state_proposal"] == "interviewing" + + +class TestCvTailorMock: + async def test_cv_tailor_returns_deterministic(self) -> None: + """cv_tailor mock returns tailored CV with change_log.""" + config = mock_config() + gw = Gateway(config) + result_a = await gw.run_task("cv_tailor", "Tailor CV for posting") + result_b = await gw.run_task("cv_tailor", "Tailor CV for posting") + assert result_a == result_b + assert "tailored_cv" in result_a + assert "change_log" in result_a + assert isinstance(result_a["change_log"], list) + assert len(result_a["change_log"]) >= 1 + # Check change_log entries have action and detail. + for entry in result_a["change_log"]: + assert "action" in entry + assert "detail" in entry + await gw.aclose() + + async def test_cv_tailor_is_strong(self) -> None: + """cv_tailor should be classified as STRONG.""" + config = mock_config() + assert config.get_task_class("cv_tailor") == TaskClass.STRONG + assert config.get_model("cv_tailor") == config.strong_model + + def test_cv_tailor_in_mock_outputs(self) -> None: + """cv_tailor should be in MOCK_OUTPUTS.""" + assert "cv_tailor" in MOCK_OUTPUTS + output = get_mock_output("cv_tailor") + assert "tailored_cv" in output + assert "change_log" in output + # Check it has skills and experience. + assert "skills" in output["tailored_cv"] + assert "experience" in output["tailored_cv"] + + +class TestDeadlineExtractMock: + async def test_deadline_extract_returns_deterministic(self) -> None: + """deadline_extract mock returns apply_by (null by default).""" + config = mock_config() + gw = Gateway(config) + result_a = await gw.run_task("deadline_extract", "Extract deadline") + result_b = await gw.run_task("deadline_extract", "Extract deadline") + assert result_a == result_b + assert "apply_by" in result_a + # Default mock has null deadline. + assert result_a["apply_by"] is None + await gw.aclose() + + async def test_deadline_extract_is_cheap(self) -> None: + """deadline_extract should be classified as CHEAP.""" + config = mock_config() + assert config.get_task_class("deadline_extract") == TaskClass.CHEAP + assert config.get_model("deadline_extract") == config.cheap_model + + def test_deadline_extract_in_mock_outputs(self) -> None: + """deadline_extract should be in MOCK_OUTPUTS.""" + assert "deadline_extract" in MOCK_OUTPUTS + output = get_mock_output("deadline_extract") + assert "apply_by" in output + assert output["apply_by"] is None \ No newline at end of file