Screenshot pipeline + fixes found under real render (CORS, batch state stomp, prompt-aware cv_tailor mock)
- scripts/screenshots.py + scripts/Dockerfile.shots: playwright-in-compose runner, 7 demo shots against the seeded stack (DinD-safe, output volume) - docker-compose.yml: api + web services (nginx SPA + /api proxy-less via build-arg VITE_API_BASE), shots service with volume output - apps/api: add CORSMiddleware (SPA origin web:80 could not read api:8000), batch scoring now refuses to rewrite applications beyond discovered/scored (kanban page load previously reset sent/interviewing to scored), prompt-aware cv_tailor mock so demo passes hallucination guard, hallucination-guard tests repointed to monkeypatched run_task - apps/web: Applications kanban batches only unscored applications, red flag display falls back to stored rationale - 159 api tests + 14 web tests green after fixes
This commit is contained in:
parent
9be1fd991b
commit
aa858bd2c1
9 changed files with 273 additions and 59 deletions
|
|
@ -161,6 +161,30 @@ MOCK_OUTPUTS: dict[str, dict[str, Any]] = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_cv_tailor(prompt: str) -> dict[str, Any]:
|
||||||
|
"""Prompt-aware mock tailor: extracts source bullets from the prompt and
|
||||||
|
rephrases them deterministically, so the result always passes the
|
||||||
|
hallucination guard (which requires traceable source overlap)."""
|
||||||
|
import re
|
||||||
|
|
||||||
|
bullets = re.findall(r'"([A-ZÅÄÖ][^"]{20,300})"', prompt)
|
||||||
|
bullets = [b for b in bullets if "{" not in b and ":" not in b][:4]
|
||||||
|
if not bullets:
|
||||||
|
bullets = ["Experienced backend developer focused on reliability"]
|
||||||
|
tailored = []
|
||||||
|
change_log = []
|
||||||
|
for b in bullets[:2]:
|
||||||
|
tailored.append(f"{b} (tailored for this posting)")
|
||||||
|
change_log.append({"action": "rephrased", "detail": f"Emphasized relevance of: {b[:60]}..."})
|
||||||
|
for b in bullets[2:]:
|
||||||
|
tailored.append(b)
|
||||||
|
change_log.append({"action": "kept", "detail": f"Retained as-is: {b[:60]}..."})
|
||||||
|
return {
|
||||||
|
"tailored_cv": {"summary": bullets[0][:160], "bullets": tailored},
|
||||||
|
"change_log": change_log,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def run_task(
|
def run_task(
|
||||||
task: str,
|
task: str,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
|
|
@ -178,6 +202,8 @@ def run_task(
|
||||||
# Mock mode
|
# Mock mode
|
||||||
time.sleep(0.01) # simulate latency
|
time.sleep(0.01) # simulate latency
|
||||||
result = MOCK_OUTPUTS.get(task, {"result": "mock"})
|
result = MOCK_OUTPUTS.get(task, {"result": "mock"})
|
||||||
|
if task == "cv_tailor":
|
||||||
|
result = _mock_cv_tailor(prompt)
|
||||||
|
|
||||||
# Validate against schema if provided (basic check)
|
# Validate against schema if provided (basic check)
|
||||||
# In real gateway this would be jsonschema validation
|
# In real gateway this would be jsonschema validation
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,18 @@ except ImportError:
|
||||||
|
|
||||||
app = FastAPI(title="Jobhunt API", version="0.2.0")
|
app = FastAPI(title="Jobhunt API", version="0.2.0")
|
||||||
|
|
||||||
|
# Self-hosted single-user app; SPA origin differs from API origin (e.g. web:80 -> api:8000).
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
|
_cors_origins = os.environ.get("CORS_ORIGINS", "*").split(",")
|
||||||
|
app.add_middleware(
|
||||||
|
CORSMiddleware,
|
||||||
|
allow_origins=[o.strip() for o in _cors_origins if o.strip()],
|
||||||
|
allow_credentials=False,
|
||||||
|
allow_methods=["*"],
|
||||||
|
allow_headers=["*"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.on_event("startup")
|
@app.on_event("startup")
|
||||||
def _startup() -> None:
|
def _startup() -> None:
|
||||||
|
|
@ -858,7 +870,13 @@ def postings_fetch(body: PostingsFetchRequest) -> Any:
|
||||||
# --- v1: Batch Scoring ---
|
# --- v1: Batch Scoring ---
|
||||||
|
|
||||||
def _batch_score_internal(application_ids: list[str]) -> list[dict[str, Any]]:
|
def _batch_score_internal(application_ids: list[str]) -> list[dict[str, Any]]:
|
||||||
"""Internal: score multiple applications, return results with red_flags."""
|
"""Internal: score multiple applications, return results with red_flags.
|
||||||
|
|
||||||
|
Safety rule (post-bug): batch scoring only touches applications in
|
||||||
|
'discovered' or 'scored' state. Anything further down the pipeline
|
||||||
|
(approved/sent/interviewing/...) keeps its state and stored scores;
|
||||||
|
those rows are returned unchanged with skipped=True.
|
||||||
|
"""
|
||||||
results: list[dict[str, Any]] = []
|
results: list[dict[str, Any]] = []
|
||||||
for app_id in application_ids:
|
for app_id in application_ids:
|
||||||
app_row = repo_app.get_application(app_id)
|
app_row = repo_app.get_application(app_id)
|
||||||
|
|
@ -868,6 +886,17 @@ def _batch_score_internal(application_ids: list[str]) -> list[dict[str, Any]]:
|
||||||
if posting is None:
|
if posting is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if app_row["state"] not in ("discovered", "scored"):
|
||||||
|
existing = app_row.get("score_rationale") or {}
|
||||||
|
results.append({
|
||||||
|
"application_id": app_id,
|
||||||
|
"score": app_row.get("score"),
|
||||||
|
"rationale": existing,
|
||||||
|
"red_flags": existing.get("red_flags", []),
|
||||||
|
"skipped": True,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
|
||||||
result = llm.run_task(
|
result = llm.run_task(
|
||||||
"score",
|
"score",
|
||||||
f"Score this posting: {posting['title']} at {posting['company']}",
|
f"Score this posting: {posting['title']} at {posting['company']}",
|
||||||
|
|
|
||||||
|
|
@ -398,15 +398,12 @@ class TestTailorCvHappyPath:
|
||||||
# ========================================================================
|
# ========================================================================
|
||||||
|
|
||||||
class TestTailorCvHallucinationGuard:
|
class TestTailorCvHallucinationGuard:
|
||||||
def test_hallucination_rejection_502(self, client):
|
def test_hallucination_rejection_502(self, client, monkeypatch):
|
||||||
"""When mock returns bullet with no source mapping, return 502."""
|
"""When the tailor output has bullets with no source mapping, return 502."""
|
||||||
app_id = _create_app_with_profile_and_sections(client)
|
|
||||||
|
|
||||||
# Patch the mock to return a fabricated bullet
|
|
||||||
import app.llm as llm_mod
|
import app.llm as llm_mod
|
||||||
original = llm_mod.MOCK_OUTPUTS.get("cv_tailor", {}).copy()
|
|
||||||
try:
|
app_id = _create_app_with_profile_and_sections(client)
|
||||||
llm_mod.MOCK_OUTPUTS["cv_tailor"] = {
|
fabricated = {
|
||||||
"tailored_cv": {
|
"tailored_cv": {
|
||||||
"summary": "Developer",
|
"summary": "Developer",
|
||||||
"skills": ["Python"],
|
"skills": ["Python"],
|
||||||
|
|
@ -422,12 +419,14 @@ class TestTailorCvHallucinationGuard:
|
||||||
},
|
},
|
||||||
"change_log": [{"action": "invented", "detail": "Made up a bullet"}],
|
"change_log": [{"action": "invented", "detail": "Made up a bullet"}],
|
||||||
}
|
}
|
||||||
|
monkeypatch.setattr(
|
||||||
|
llm_mod, "run_task",
|
||||||
|
lambda task, prompt, *a, **k: fabricated,
|
||||||
|
)
|
||||||
resp = client.post(f"/api/applications/{app_id}/tailor-cv")
|
resp = client.post(f"/api/applications/{app_id}/tailor-cv")
|
||||||
assert resp.status_code == 502
|
assert resp.status_code == 502
|
||||||
detail = resp.json()["detail"]
|
detail = resp.json()["detail"]
|
||||||
assert "hallucination_guard" in str(detail)
|
assert "hallucination_guard" in str(detail)
|
||||||
finally:
|
|
||||||
llm_mod.MOCK_OUTPUTS["cv_tailor"] = original
|
|
||||||
|
|
||||||
def test_hallucination_rejection_with_empty_source_bullets(self, client):
|
def test_hallucination_rejection_with_empty_source_bullets(self, client):
|
||||||
"""When there are no source bullets, hallucination guard is not triggered (no source to map to)."""
|
"""When there are no source bullets, hallucination guard is not triggered (no source to map to)."""
|
||||||
|
|
@ -456,15 +455,12 @@ class TestTailorCvHallucinationGuard:
|
||||||
# Should succeed since source_bullets is empty -> guard not triggered
|
# Should succeed since source_bullets is empty -> guard not triggered
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
||||||
def test_hallucination_rejection_preserves_existing_output(self, client):
|
def test_hallucination_rejection_preserves_existing_output(self, client, monkeypatch):
|
||||||
"""After a 502 hallucination rejection, a subsequent valid call works."""
|
"""After a 502 hallucination rejection, a subsequent valid call works."""
|
||||||
app_id = _create_app_with_profile_and_sections(client)
|
|
||||||
|
|
||||||
# First: trigger hallucination
|
|
||||||
import app.llm as llm_mod
|
import app.llm as llm_mod
|
||||||
original = llm_mod.MOCK_OUTPUTS.get("cv_tailor", {}).copy()
|
|
||||||
try:
|
app_id = _create_app_with_profile_and_sections(client)
|
||||||
llm_mod.MOCK_OUTPUTS["cv_tailor"] = {
|
fabricated = {
|
||||||
"tailored_cv": {
|
"tailored_cv": {
|
||||||
"summary": "Dev",
|
"summary": "Dev",
|
||||||
"skills": ["Python"],
|
"skills": ["Python"],
|
||||||
|
|
@ -478,15 +474,17 @@ class TestTailorCvHallucinationGuard:
|
||||||
},
|
},
|
||||||
"change_log": [],
|
"change_log": [],
|
||||||
}
|
}
|
||||||
|
with monkeypatch.context() as mp:
|
||||||
|
mp.setattr(
|
||||||
|
llm_mod, "run_task",
|
||||||
|
lambda task, prompt, *a, **k: fabricated,
|
||||||
|
)
|
||||||
resp1 = client.post(f"/api/applications/{app_id}/tailor-cv")
|
resp1 = client.post(f"/api/applications/{app_id}/tailor-cv")
|
||||||
assert resp1.status_code == 502
|
assert resp1.status_code == 502
|
||||||
|
|
||||||
# Restore and retry
|
# Default prompt-aware mock is guard-safe -> succeeds
|
||||||
llm_mod.MOCK_OUTPUTS["cv_tailor"] = original
|
|
||||||
resp2 = client.post(f"/api/applications/{app_id}/tailor-cv")
|
resp2 = client.post(f"/api/applications/{app_id}/tailor-cv")
|
||||||
assert resp2.status_code == 200
|
assert resp2.status_code == 200
|
||||||
finally:
|
|
||||||
llm_mod.MOCK_OUTPUTS["cv_tailor"] = original
|
|
||||||
|
|
||||||
|
|
||||||
# ========================================================================
|
# ========================================================================
|
||||||
|
|
|
||||||
14
apps/web/Dockerfile
Normal file
14
apps/web/Dockerfile
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Web production image: build the SPA, serve via nginx with SPA fallback
|
||||||
|
FROM node:22-alpine AS build
|
||||||
|
WORKDIR /w
|
||||||
|
ARG VITE_API_BASE=http://api:8000/api
|
||||||
|
ENV VITE_API_BASE=$VITE_API_BASE
|
||||||
|
COPY apps/web/package.json apps/web/package-lock.json ./
|
||||||
|
RUN npm ci --no-audit --no-fund
|
||||||
|
COPY apps/web ./
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
FROM nginx:1.27-alpine
|
||||||
|
COPY apps/web/nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
COPY --from=build /w/dist /usr/share/nginx/html
|
||||||
|
EXPOSE 80
|
||||||
15
apps/web/nginx.conf
Normal file
15
apps/web/nginx.conf
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name _;
|
||||||
|
root /usr/share/nginx/html;
|
||||||
|
index index.html;
|
||||||
|
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass http://api:8000/api/;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
try_files $uri $uri/ /index.html;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -41,7 +41,10 @@ function hasRedFlags(app: Application): boolean {
|
||||||
}
|
}
|
||||||
|
|
||||||
function redFlagsFor(app: Application): string[] {
|
function redFlagsFor(app: Application): string[] {
|
||||||
return redFlagsMap.value[app.id] ?? []
|
const fromBatch = redFlagsMap.value[app.id]
|
||||||
|
if (fromBatch) return fromBatch
|
||||||
|
const stored = (app.score_rationale as { red_flags?: string[] } | null)?.red_flags
|
||||||
|
return stored ?? []
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasNudge(app: Application): boolean {
|
function hasNudge(app: Application): boolean {
|
||||||
|
|
@ -53,7 +56,11 @@ async function loadApplications() {
|
||||||
applications.value = await api.getApplications()
|
applications.value = await api.getApplications()
|
||||||
// Load red flags via batch scoring and nudges via today endpoint
|
// Load red flags via batch scoring and nudges via today endpoint
|
||||||
const [batchResult, todayResult] = await Promise.allSettled([
|
const [batchResult, todayResult] = await Promise.allSettled([
|
||||||
api.batchScore(applications.value.map((a) => a.id)),
|
api.batchScore(
|
||||||
|
applications.value
|
||||||
|
.filter((a) => a.state === 'discovered' || a.state === 'scored')
|
||||||
|
.map((a) => a.id)
|
||||||
|
),
|
||||||
api.getToday()
|
api.getToday()
|
||||||
])
|
])
|
||||||
if (batchResult.status === 'fulfilled') {
|
if (batchResult.status === 'fulfilled') {
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,42 @@ services:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
restart: "no"
|
restart: "no"
|
||||||
|
|
||||||
|
api:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: apps/api/Dockerfile.test
|
||||||
|
image: jobhunt-platform-api-test
|
||||||
|
entrypoint: uvicorn app.main:app --host 0.0.0.0 --port 8000
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql://jobhunt:jobhunt@postgres:5432/jobhunt
|
||||||
|
working_dir: /app/apps/api
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
web:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: apps/web/Dockerfile
|
||||||
|
args:
|
||||||
|
VITE_API_BASE: http://api:8000/api
|
||||||
|
depends_on:
|
||||||
|
- api
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
shots:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: scripts/Dockerfile.shots
|
||||||
|
volumes:
|
||||||
|
- shots_out:/out
|
||||||
|
depends_on:
|
||||||
|
- api
|
||||||
|
- web
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
jobhunt_pgdata:
|
jobhunt_pgdata:
|
||||||
name: jobhunt_pgdata
|
name: jobhunt_pgdata
|
||||||
|
shots_out:
|
||||||
5
scripts/Dockerfile.shots
Normal file
5
scripts/Dockerfile.shots
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
FROM mcr.microsoft.com/playwright/python:v1.55.0-jammy
|
||||||
|
WORKDIR /shots
|
||||||
|
RUN pip install --no-cache-dir playwright==1.55.0
|
||||||
|
COPY scripts/screenshots.py ./
|
||||||
|
CMD ["python", "screenshots.py"]
|
||||||
84
scripts/screenshots.py
Normal file
84
scripts/screenshots.py
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
"""Screenshot runner: drives the seed demo UI and saves PNGs to /out.
|
||||||
|
|
||||||
|
Runs inside the compose network; browser resolves 'web' and 'api' directly.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from playwright.sync_api import sync_playwright
|
||||||
|
|
||||||
|
BASE = "http://web"
|
||||||
|
API = "http://api:8000/api"
|
||||||
|
OUT = "/out"
|
||||||
|
|
||||||
|
|
||||||
|
def api(method, path, body=None):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
API + path,
|
||||||
|
data=json.dumps(body).encode() if body else None,
|
||||||
|
method=method,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
return json.loads(r.read())
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# Idempotent: ensures full demo dataset exists
|
||||||
|
seed = api("POST", "/concierge/seed-demo")
|
||||||
|
print("seed:", json.dumps(seed)[:200])
|
||||||
|
|
||||||
|
apps = api("GET", "/applications")
|
||||||
|
interview = next((a for a in apps if a["state"] == "interviewing"), apps[0])
|
||||||
|
detail_id = interview["id"]
|
||||||
|
print("detail id:", detail_id)
|
||||||
|
|
||||||
|
shots = [
|
||||||
|
("welcome", "/", {}), # first-run wizard may redirect away; force below
|
||||||
|
("welcome", "/welcome", {}),
|
||||||
|
("today", "/today", {}),
|
||||||
|
("cv", "/cv", {}),
|
||||||
|
("research", "/research", {}),
|
||||||
|
("applications", "/applications", {}),
|
||||||
|
("detail", f"/applications/{detail_id}", {}),
|
||||||
|
]
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch(args=["--disable-dev-shm-usage"])
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 900},
|
||||||
|
device_scale_factor=2)
|
||||||
|
for name, path, _opts in shots[1:]: # skip the "/" duplicate
|
||||||
|
try:
|
||||||
|
page.goto(BASE + path, wait_until="networkidle", timeout=30000)
|
||||||
|
page.wait_for_timeout(1200)
|
||||||
|
# close possible wizard redirect back
|
||||||
|
if name != "welcome" and page.url.endswith("/welcome"):
|
||||||
|
page.goto(BASE + path, wait_until="networkidle", timeout=30000)
|
||||||
|
page.wait_for_timeout(800)
|
||||||
|
page.screenshot(path=f"{OUT}/{name}.png")
|
||||||
|
print("shot:", name, "<-", page.url)
|
||||||
|
except Exception as e:
|
||||||
|
print("FAILED:", name, type(e).__name__, str(e)[:150])
|
||||||
|
|
||||||
|
# Interaction shot: open the Tailor CV panel on an approved application
|
||||||
|
approved = next((a for a in apps if a["state"] in ("approved", "interviewing")), apps[0])
|
||||||
|
try:
|
||||||
|
page.goto(f"{BASE}/applications/{approved['id']}", wait_until="networkidle", timeout=30000)
|
||||||
|
page.wait_for_timeout(1000)
|
||||||
|
btn = page.locator('[data-testid="tailor-cv-btn"]')
|
||||||
|
btn.click(timeout=8000)
|
||||||
|
page.wait_for_selector('[data-testid="tailor-panel"]', timeout=20000)
|
||||||
|
page.wait_for_timeout(800)
|
||||||
|
page.screenshot(path=f"{OUT}/detail-tailor.png")
|
||||||
|
print("shot: detail-tailor")
|
||||||
|
except Exception as e:
|
||||||
|
print("FAILED: detail-tailor", type(e).__name__, str(e)[:150])
|
||||||
|
|
||||||
|
browser.close()
|
||||||
|
print("DONE")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Loading…
Reference in a new issue