Codex
139 lines
5.4 KiB
Python
139 lines
5.4 KiB
Python
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCRIPT = ROOT / "skills/codex-subagent-router/scripts/validate_evidence_packet.py"
|
|
spec = importlib.util.spec_from_file_location("evidence", SCRIPT)
|
|
evidence = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(evidence)
|
|
|
|
|
|
def fixture(name="completed-unknown"):
|
|
return json.loads((ROOT / "examples" / (name + ".json")).read_text())
|
|
|
|
|
|
class EvidenceTests(unittest.TestCase):
|
|
def test_all_packaged_examples(self):
|
|
for path in (ROOT / "examples").glob("*.json"):
|
|
with self.subTest(path=path.name):
|
|
evidence.validate(json.loads(path.read_text()))
|
|
|
|
def test_completed_unknown_identity_is_valid(self):
|
|
packet = fixture()
|
|
evidence.validate(packet)
|
|
self.assertEqual(packet["tool_receipts"][0]["observed"]["model"], "unknown")
|
|
|
|
def test_known_identity_requires_host_evidence(self):
|
|
packet = fixture()
|
|
child = packet["tool_receipts"][0]
|
|
child["observed"]["model"] = "gpt-5.6-terra"
|
|
with self.assertRaisesRegex(ValueError, "host evidence"):
|
|
evidence.validate(packet)
|
|
child["identity_source"] = "host-tool-result"
|
|
child["identity_evidence"] = "synthetic host event for parser testing"
|
|
evidence.validate(packet)
|
|
|
|
def test_child_self_description_is_not_identity_source(self):
|
|
packet = fixture()
|
|
packet["tool_receipts"][0]["identity_source"] = "child-self-description"
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
|
|
def test_child_execution_requires_actual_child_id(self):
|
|
packet = fixture()
|
|
del packet["tool_receipts"][0]["child_id"]
|
|
with self.assertRaisesRegex(ValueError, "child_id"):
|
|
evidence.validate(packet)
|
|
|
|
def test_child_execution_without_receipt_fails(self):
|
|
packet = fixture()
|
|
packet["tool_receipts"] = []
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
|
|
def test_not_run_cannot_claim_child_or_changes(self):
|
|
packet = fixture()
|
|
packet["execution"] = "not_run"
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
packet = fixture("not-run")
|
|
packet["changes"] = ["synthetic.py"]
|
|
with self.assertRaisesRegex(ValueError, "changes"):
|
|
evidence.validate(packet)
|
|
|
|
def test_running_receipt_does_not_need_completion(self):
|
|
packet = fixture()
|
|
packet["tool_receipts"][0]["status"] = "running"
|
|
evidence.validate(packet)
|
|
|
|
def test_schema_cannot_masquerade_as_child(self):
|
|
packet = fixture()
|
|
packet["tool_receipts"][0]["kind"] = "schema"
|
|
with self.assertRaisesRegex(ValueError, "child_id"):
|
|
evidence.validate(packet)
|
|
|
|
def test_invalid_unhashable_sources_raise_validation_error(self):
|
|
for field in ("capability_source", "execution"):
|
|
packet = fixture()
|
|
packet[field] = []
|
|
with self.subTest(field=field), self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
|
|
def test_malformed_receipts_and_identity_raise_validation_error(self):
|
|
for value in (None, [], "bad", 3):
|
|
packet = fixture()
|
|
packet["tool_receipts"] = [value]
|
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
packet = fixture()
|
|
packet["tool_receipts"][0]["observed"] = []
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
|
|
def test_failed_command_cannot_claim_pass(self):
|
|
packet = fixture()
|
|
packet["verification"]["commands"] = [
|
|
{"command": "synthetic-check", "exit_code": 1, "status": "passed"}]
|
|
with self.assertRaisesRegex(ValueError, "exit_code 0"):
|
|
evidence.validate(packet)
|
|
|
|
def test_boolean_is_not_exit_code_or_schema_version(self):
|
|
packet = fixture()
|
|
packet["verification"]["commands"] = [
|
|
{"command": "synthetic-check", "exit_code": False, "status": "passed"}]
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
packet = fixture()
|
|
packet["schema_version"] = True
|
|
with self.assertRaises(ValueError):
|
|
evidence.validate(packet)
|
|
|
|
def test_escalation_needs_target_and_reason(self):
|
|
packet = fixture()
|
|
packet["escalate"] = {"required": True}
|
|
with self.assertRaisesRegex(ValueError, "target and reason"):
|
|
evidence.validate(packet)
|
|
|
|
def test_legacy_is_still_readable_with_notice(self):
|
|
result = subprocess.run([sys.executable, str(SCRIPT), str(ROOT / "examples/legacy-v1.json")],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("legacy v1", result.stderr)
|
|
|
|
def test_cli_stdin_and_bad_json(self):
|
|
good = subprocess.run([sys.executable, str(SCRIPT), "-"], input=json.dumps(fixture()),
|
|
capture_output=True, text=True)
|
|
self.assertEqual(good.returncode, 0, good.stderr)
|
|
bad = subprocess.run([sys.executable, str(SCRIPT), "-"], input="{",
|
|
capture_output=True, text=True)
|
|
self.assertEqual(bad.returncode, 1)
|
|
self.assertNotIn("Traceback", bad.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|