Add IT-One AI agent template, eval gate, and deployment skeleton

2026-08-21 12:14:55 +02:00
parent d1a12d7880
commit d406da1312
25 changed files with 243 additions and 2 deletions
@@ -0,0 +1,8 @@
+def score_case(case: dict, result: dict) -> tuple[bool, str]:
+    answer = str(result.get("answer", ""))
+    expected = str(case.get("expected_answer_contains", ""))
+    if expected and expected.lower() not in answer.lower():
+        return False, "answer substring mismatch"
+    if len(result.get("tool_calls", [])) != int(case.get("expected_tool_calls", 0)):
+        return False, "tool call count mismatch"
+    return True, "ok"