Add IT-One AI agent template, eval gate, and deployment skeleton

evals/assert_rules.py

@@ -0,0 +1,8 @@
+def score_case(case: dict, result: dict) -> tuple[bool, str]:
+    answer = str(result.get("answer", ""))
+    expected = str(case.get("expected_answer_contains", ""))
+    if expected and expected.lower() not in answer.lower():
+        return False, "answer substring mismatch"
+    if len(result.get("tool_calls", [])) != int(case.get("expected_tool_calls", 0)):
+        return False, "tool call count mismatch"
+    return True, "ok"

evals/run_evals.py

@@ -0,0 +1,20 @@
+import argparse, json, pathlib, sys
+from src.core.agent import handle
+from evals.assert_rules import score_case
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--threshold", type=float, default=0.95)
+    args = parser.parse_args()
+    cases = [json.loads(line) for line in pathlib.Path("evals/test_cases.jsonl").read_text().splitlines() if line.strip()]
+    passed = 0
+    for case in cases:
+        ok, reason = score_case(case, handle(case["input"]))
+        passed += int(ok)
+        print(f"{case['id']}: {'PASS' if ok else 'FAIL'} ({reason})")
+    score = passed / len(cases) if cases else 0.0
+    print(f"accuracy={score:.3f} threshold={args.threshold:.3f}")
+    return 0 if score >= args.threshold else 1
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/test_cases.jsonl

@@ -0,0 +1,5 @@
+{"id":"greeting","input":"Hello","expected_answer_contains":"Hello","expected_tool_calls":0}
+{"id":"empty-safe","input":"  ","expected_answer_contains":"","expected_tool_calls":0}
+{"id":"explicit-request","input":"List the next step","expected_answer_contains":"List","expected_tool_calls":0}
+{"id":"no-invention","input":"Return the supplied value","expected_answer_contains":"supplied","expected_tool_calls":0}
+{"id":"traceable","input":"Show the request","expected_answer_contains":"request","expected_tool_calls":0}