Bootstrap model routing skill, prompt, evidence validation and development docs
This commit is contained in:
Codex
@@ -0,0 +1,138 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "skills/codex-subagent-router/scripts/validate_evidence_packet.py"
|
||||
spec = importlib.util.spec_from_file_location("evidence", SCRIPT)
|
||||
evidence = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(evidence)
|
||||
|
||||
|
||||
def fixture(name="completed-unknown"):
|
||||
return json.loads((ROOT / "examples" / (name + ".json")).read_text())
|
||||
|
||||
|
||||
class EvidenceTests(unittest.TestCase):
|
||||
def test_all_packaged_examples(self):
|
||||
for path in (ROOT / "examples").glob("*.json"):
|
||||
with self.subTest(path=path.name):
|
||||
evidence.validate(json.loads(path.read_text()))
|
||||
|
||||
def test_completed_unknown_identity_is_valid(self):
|
||||
packet = fixture()
|
||||
evidence.validate(packet)
|
||||
self.assertEqual(packet["tool_receipts"][0]["observed"]["model"], "unknown")
|
||||
|
||||
def test_known_identity_requires_host_evidence(self):
|
||||
packet = fixture()
|
||||
child = packet["tool_receipts"][0]
|
||||
child["observed"]["model"] = "gpt-5.6-terra"
|
||||
with self.assertRaisesRegex(ValueError, "host evidence"):
|
||||
evidence.validate(packet)
|
||||
child["identity_source"] = "host-tool-result"
|
||||
child["identity_evidence"] = "synthetic host event for parser testing"
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_child_self_description_is_not_identity_source(self):
|
||||
packet = fixture()
|
||||
packet["tool_receipts"][0]["identity_source"] = "child-self-description"
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_child_execution_requires_actual_child_id(self):
|
||||
packet = fixture()
|
||||
del packet["tool_receipts"][0]["child_id"]
|
||||
with self.assertRaisesRegex(ValueError, "child_id"):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_child_execution_without_receipt_fails(self):
|
||||
packet = fixture()
|
||||
packet["tool_receipts"] = []
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_not_run_cannot_claim_child_or_changes(self):
|
||||
packet = fixture()
|
||||
packet["execution"] = "not_run"
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
packet = fixture("not-run")
|
||||
packet["changes"] = ["synthetic.py"]
|
||||
with self.assertRaisesRegex(ValueError, "changes"):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_running_receipt_does_not_need_completion(self):
|
||||
packet = fixture()
|
||||
packet["tool_receipts"][0]["status"] = "running"
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_schema_cannot_masquerade_as_child(self):
|
||||
packet = fixture()
|
||||
packet["tool_receipts"][0]["kind"] = "schema"
|
||||
with self.assertRaisesRegex(ValueError, "child_id"):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_invalid_unhashable_sources_raise_validation_error(self):
|
||||
for field in ("capability_source", "execution"):
|
||||
packet = fixture()
|
||||
packet[field] = []
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_malformed_receipts_and_identity_raise_validation_error(self):
|
||||
for value in (None, [], "bad", 3):
|
||||
packet = fixture()
|
||||
packet["tool_receipts"] = [value]
|
||||
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
packet = fixture()
|
||||
packet["tool_receipts"][0]["observed"] = []
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_failed_command_cannot_claim_pass(self):
|
||||
packet = fixture()
|
||||
packet["verification"]["commands"] = [
|
||||
{"command": "synthetic-check", "exit_code": 1, "status": "passed"}]
|
||||
with self.assertRaisesRegex(ValueError, "exit_code 0"):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_boolean_is_not_exit_code_or_schema_version(self):
|
||||
packet = fixture()
|
||||
packet["verification"]["commands"] = [
|
||||
{"command": "synthetic-check", "exit_code": False, "status": "passed"}]
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
packet = fixture()
|
||||
packet["schema_version"] = True
|
||||
with self.assertRaises(ValueError):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_escalation_needs_target_and_reason(self):
|
||||
packet = fixture()
|
||||
packet["escalate"] = {"required": True}
|
||||
with self.assertRaisesRegex(ValueError, "target and reason"):
|
||||
evidence.validate(packet)
|
||||
|
||||
def test_legacy_is_still_readable_with_notice(self):
|
||||
result = subprocess.run([sys.executable, str(SCRIPT), str(ROOT / "examples/legacy-v1.json")],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("legacy v1", result.stderr)
|
||||
|
||||
def test_cli_stdin_and_bad_json(self):
|
||||
good = subprocess.run([sys.executable, str(SCRIPT), "-"], input=json.dumps(fixture()),
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(good.returncode, 0, good.stderr)
|
||||
bad = subprocess.run([sys.executable, str(SCRIPT), "-"], input="{",
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(bad.returncode, 1)
|
||||
self.assertNotIn("Traceback", bad.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,65 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("installer", ROOT / "scripts/install.py")
|
||||
installer = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(installer)
|
||||
|
||||
|
||||
class InstallerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.home = Path(self.temp.name) / "codex-home"
|
||||
self.home.mkdir()
|
||||
|
||||
def test_preview_never_writes(self):
|
||||
result = installer.install(ROOT, self.home, include_prompt=True)
|
||||
self.assertTrue(result["changed_paths"])
|
||||
self.assertFalse(result["applied"])
|
||||
self.assertEqual(list(self.home.iterdir()), [])
|
||||
|
||||
def test_apply_preserves_user_files_and_backs_up(self):
|
||||
prompt = self.home / "AGENTS.md"
|
||||
prompt.write_text("user prompt\n")
|
||||
config = self.home / "config.toml"
|
||||
config.write_text('model = "user-selected"\n')
|
||||
skill = self.home / "skills/codex-subagent-router"
|
||||
skill.mkdir(parents=True)
|
||||
custom = skill / "custom.md"
|
||||
custom.write_text("local extension")
|
||||
result = installer.install(ROOT, self.home, include_prompt=True, apply=True)
|
||||
self.assertEqual((Path(result["backup"]) / "AGENTS.md").read_text(), "user prompt\n")
|
||||
self.assertEqual(config.read_text(), 'model = "user-selected"\n')
|
||||
self.assertEqual(custom.read_text(), "local extension")
|
||||
self.assertEqual(prompt.read_bytes(), (ROOT / "prompts/AGENTS.md").read_bytes())
|
||||
again = installer.install(ROOT, self.home, include_prompt=True, apply=True)
|
||||
self.assertEqual(again["changed_paths"], [])
|
||||
self.assertIsNone(again["backup"])
|
||||
|
||||
def test_skill_only_does_not_replace_global_prompt(self):
|
||||
prompt = self.home / "AGENTS.md"
|
||||
prompt.write_text("keep")
|
||||
installer.install(ROOT, self.home, apply=True)
|
||||
self.assertEqual(prompt.read_text(), "keep")
|
||||
|
||||
def test_symlink_target_is_rejected(self):
|
||||
outside = Path(self.temp.name) / "outside"
|
||||
outside.mkdir()
|
||||
(self.home / "skills").symlink_to(outside, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symlink"):
|
||||
installer.install(ROOT, self.home, apply=True)
|
||||
self.assertEqual(list(outside.iterdir()), [])
|
||||
|
||||
def test_conflicting_directory_is_rejected_before_writes(self):
|
||||
(self.home / "AGENTS.md").mkdir()
|
||||
with self.assertRaisesRegex(ValueError, "regular file"):
|
||||
installer.install(ROOT, self.home, include_prompt=True, apply=True)
|
||||
self.assertFalse((self.home / "skills").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user