"""tests/regression/test_fallback_acceptance_2553plus58.py task-2553+58 (TRACK C) — FALLBACK_ACCEPTANCE_CRITERION_FOR_NEXT_PILOT regression. Spec: memory/tasks/task-2553+58.md (sha256 976d904dbe05971fad694ca0409d0cdbcd6cdc31434ac156371770ed9ea40f71). 회장 §3 필수 regression — 실 entrypoint 직접 호출 (mock-only FAIL): 1 criterion (a) cancel-on-success 제거 -> OPERATIONAL_PASS [a] 2 criterion (b) NON_BLOCKING schema-valid 마크 -> OPERATIONAL_PASS [b] 3 a·b 동시 -> OPERATIONAL_PASS [a,b] 4 마크 없이 발화 + DUPLICATE_CALLBACK_IGNORED-only -> OPERATIONAL_QUALITY_FAIL (회장 §2 anti_pattern — 안전성 OK 지만 운영 품질 PASS 아님) 5 마크 없이 발화 + NO-ACTION-only -> OPERATIONAL_QUALITY_FAIL 6 durable-success 미선행 -> NOT_APPLICABLE 7 bound fallback 부재 -> NOT_APPLICABLE 8 normal_success_unchanged=false -> SAFETY_FAIL (fail-closed) 9 schema-valid 마크지만 task_id binding 불일치 -> 기준 (b) 미충족 FAIL 10 비-ANU owner 마크 -> schema invalid -> 기준 (b) 미충족 11 잘못된 classification 마크 -> schema invalid -> 기준 (b) 미충족 12 criteria <-> schema 정합 (실 산출물) coherent=true 13 non_blocking schema 가 valid draft-07 14 criteria JSON 이 실 schema·실 entrypoint 를 가리킴 15 MOCK-ONLY FAIL: entrypoint 가 실 모듈/실 산출물(sha256 일치)에서 옴 16 기존 task-2553 frozen anchor byte-0 (read-only, 실행 전후 불변) 17 git HEAD/branch 불변 모든 테스트 100% offline — network / git mutation / cron / dispatch / cokacdir / subprocess / 파일 write 0. evaluate_fallback_acceptance 는 실 entrypoint 직접 호출이며 fixture 가 mock 으로 치환되면 sha256 가드(test 15) 가 실패한다. """ from __future__ import annotations import hashlib import importlib.util import json import subprocess import sys from pathlib import Path import pytest WORKSPACE = Path(__file__).resolve().parent.parent.parent SCRIPT_PATH = WORKSPACE / "scripts/validate_fallback_acceptance_2553plus58.py" CRITERIA_PATH = WORKSPACE / "memory/events/fallback_acceptance_criteria.json" SCHEMA_PATH = WORKSPACE / "schemas/non_blocking_fallback_schema.json" FIX_PATH = WORKSPACE / "memory/fixtures/task-2553plus58.cases.json" FROZEN_ANCHORS = [ "memory/events/task-2553.legacy-pending-fallback-inventory_260518.json", "memory/events/task-2553+37.fallback-duplicate-callback-ignored_260518.json", "memory/events/callback_4tuple_index.jsonl", ] def _load_real_entrypoint_module(): """Load the REAL validator module from the workspace scripts path. No mock/stub substitution — the module object is loaded straight from SCRIPT_PATH so ``__file__`` is asserted to be the real artifact (test 15). """ spec = importlib.util.spec_from_file_location( "validate_fallback_acceptance_2553plus58", SCRIPT_PATH ) assert spec and spec.loader mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod M = _load_real_entrypoint_module() FIX = json.loads(FIX_PATH.read_text(encoding="utf-8")) def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _case(name: str) -> dict: for c in FIX["observation_cases"]: if c["name"] == name: return c raise KeyError(name) def _obs(name: str) -> dict: """Resolve an observation case, expanding *_ref pointers from FIX.""" c = _case(name) if "observation" in c: obs = dict(c["observation"]) ref = obs.pop("registry_non_blocking_mark_ref", None) if ref: obs["registry_non_blocking_mark"] = FIX[ref] return obs # observation_ref cases (criterion_b / both) base = { "task_id": "task-2553+58", "fallback_cron_id": "F0683510-FB", "fallback_bound": True, "normal_callback_durable_success": True, "normal_success_unchanged": True, } if name == "criterion_b_nonblocking_mark": base["registry_non_blocking_mark"] = FIX["valid_non_blocking_mark"] elif name == "both_a_and_b": base["cancel_on_success_applied"] = True base["fallback_fired"] = False base["registry_non_blocking_mark"] = FIX["valid_non_blocking_mark"] return base # ── 1-3 PASS paths ────────────────────────────────────────────────────────── def test_01_criterion_a_cancel_removed_pass(): v = M.evaluate_fallback_acceptance(_obs("criterion_a_cancel_removed")) assert v["verdict"] == M.OPERATIONAL_PASS assert v["satisfied_criterion"] == ["a"] def test_02_criterion_b_nonblocking_mark_pass(): v = M.evaluate_fallback_acceptance(_obs("criterion_b_nonblocking_mark")) assert v["verdict"] == M.OPERATIONAL_PASS assert v["satisfied_criterion"] == ["b"] def test_03_both_a_and_b_pass(): v = M.evaluate_fallback_acceptance(_obs("both_a_and_b")) assert v["verdict"] == M.OPERATIONAL_PASS assert v["satisfied_criterion"] == ["a", "b"] # ── 4-5 the §2 anti-pattern: DUPLICATE/NO-ACTION only, no mark ────────────── def test_04_dup_ignored_only_no_mark_is_quality_fail(): v = M.evaluate_fallback_acceptance(_obs("dup_ignored_only_no_mark")) assert v["verdict"] == M.OPERATIONAL_QUALITY_FAIL assert v["reason"] == "DUPLICATE_IGNORED_ONLY_NO_MARK" # 안전성은 OK 임을 명시 (운영 품질만 FAIL) assert "OK" in v["safety"] def test_05_no_action_only_no_mark_is_quality_fail(): v = M.evaluate_fallback_acceptance(_obs("no_action_only_no_mark")) assert v["verdict"] == M.OPERATIONAL_QUALITY_FAIL assert v["reason"] == "DUPLICATE_IGNORED_ONLY_NO_MARK" # ── 6-8 applicability / safety gate ───────────────────────────────────────── def test_06_not_applicable_no_durable_success(): v = M.evaluate_fallback_acceptance(_obs("not_applicable_no_durable_success")) assert v["verdict"] == M.NOT_APPLICABLE def test_07_not_applicable_no_bound_fallback(): v = M.evaluate_fallback_acceptance(_obs("not_applicable_no_bound_fallback")) assert v["verdict"] == M.NOT_APPLICABLE def test_08_safety_fail_when_decouple_violated(): v = M.evaluate_fallback_acceptance(_obs("safety_fail_decouple_violated")) assert v["verdict"] == M.SAFETY_FAIL assert v["reason"] == "NORMAL_SUCCESS_DECOUPLE_VIOLATED" # ── 9-11 criterion (b) machinery: binding + schema strictness ────────────── def test_09_mark_task_id_mismatch_not_criterion_b(): v = M.evaluate_fallback_acceptance(_obs("mark_present_but_task_id_mismatch")) assert v["verdict"] == M.OPERATIONAL_QUALITY_FAIL assert v["criterion_b"] is False assert v["mark_binding_ok"] is False def test_10_non_anu_owner_mark_schema_invalid(): ok, errs = M.validate_non_blocking_mark(FIX["malformed_mark_non_anu_owner"]) assert ok is False and errs # 같은 마크를 단 observation 은 기준 (b) 미충족 obs = { "task_id": "task-2553+58", "fallback_cron_id": "F0683510-FB", "fallback_bound": True, "normal_callback_durable_success": True, "normal_success_unchanged": True, "cancel_on_success_applied": False, "fallback_fired": True, "fallback_handling": "DUPLICATE_CALLBACK_IGNORED", "registry_non_blocking_mark": FIX["malformed_mark_non_anu_owner"], } v = M.evaluate_fallback_acceptance(obs) assert v["verdict"] == M.OPERATIONAL_QUALITY_FAIL assert v["criterion_b"] is False def test_11_bad_classification_mark_schema_invalid(): ok, errs = M.validate_non_blocking_mark( FIX["malformed_mark_bad_classification"] ) assert ok is False and errs # ── 12-14 criteria/schema 정합 (실 산출물) ────────────────────────────────── def test_12_criteria_schema_coherent(): c = M.check_criteria_schema_coherence() assert c["coherent"] is True, c["checks"] assert all(c["checks"].values()) def test_13_non_blocking_schema_is_valid_draft07(): from jsonschema import Draft7Validator # pyright: ignore[reportMissingImports] schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) Draft7Validator.check_schema(schema) # raises if invalid assert schema["$id"] == "schemas/non_blocking_fallback_schema.json" assert schema["additionalProperties"] is False def test_14_criteria_points_to_real_schema_and_entrypoint(): crit = json.loads(CRITERIA_PATH.read_text(encoding="utf-8")) assert ( crit["criterion_b"]["schema_ref"] == "schemas/non_blocking_fallback_schema.json" ) ep = crit["entrypoint"] assert ep["module"] == "scripts/validate_fallback_acceptance_2553plus58.py" assert ep["function"] == "evaluate_fallback_acceptance" assert hasattr(M, "evaluate_fallback_acceptance") assert hasattr(M, "check_criteria_schema_coherence") assert hasattr(M, "validate_non_blocking_mark") # ── 15 MOCK-ONLY FAIL guard ───────────────────────────────────────────────── def test_15_real_entrypoint_not_mock(): # 모듈이 실제 workspace scripts 경로에서 로드됐는지. mod_file = M.__file__ assert mod_file is not None assert Path(mod_file).resolve() == SCRIPT_PATH.resolve() # entrypoint 가 그 모듈에서 정의됐는지 (monkeypatch 된 mock 아님). assert M.evaluate_fallback_acceptance.__module__ == M.__name__ # 실 산출물을 소비하는지 — module 상수 경로가 실파일이고 sha256 가 # disk 와 일치 (fixture 가 stub 으로 치환되면 불일치). assert M.CRITERIA_PATH.resolve() == CRITERIA_PATH.resolve() assert M.SCHEMA_PATH.resolve() == SCHEMA_PATH.resolve() assert M.sha256_of(CRITERIA_PATH) == _sha(CRITERIA_PATH) assert M.sha256_of(SCHEMA_PATH) == _sha(SCHEMA_PATH) # 실 CLI 가 read-only(파일 write 0) 로 동작. rc = M.main([]) assert rc == 0 # ── 16-17 invariants ──────────────────────────────────────────────────────── def test_16_frozen_anchors_byte0_read_only(): pre = {a: _sha(WORKSPACE / a) for a in FROZEN_ANCHORS} # 실 entrypoint·CLI·coherence 를 모두 호출 (read-only 임을 실증). M.evaluate_fallback_acceptance(_obs("dup_ignored_only_no_mark")) M.check_criteria_schema_coherence() M.main([]) post = {a: _sha(WORKSPACE / a) for a in FROZEN_ANCHORS} assert pre == post def test_17_git_head_branch_unchanged(): head = subprocess.run( ["git", "-C", str(WORKSPACE), "rev-parse", "HEAD"], capture_output=True, text=True, check=True, ).stdout.strip() branch = subprocess.run( ["git", "-C", str(WORKSPACE), "branch", "--show-current"], capture_output=True, text=True, check=True, ).stdout.strip() assert head == "20456b5f83fc039f2fd6f50f4b94095c29b41bfb" assert branch == "task/task-2553p1-f1-clean-replacement" if __name__ == "__main__": sys.exit(pytest.main([__file__, "-q"]))