Template
21 lines
805 B
Python
21 lines
805 B
Python
import argparse, json, pathlib, sys
|
|
from src.core.agent import handle
|
|
from evals.assert_rules import score_case
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--threshold", type=float, default=0.95)
|
|
args = parser.parse_args()
|
|
cases = [json.loads(line) for line in pathlib.Path("evals/test_cases.jsonl").read_text().splitlines() if line.strip()]
|
|
passed = 0
|
|
for case in cases:
|
|
ok, reason = score_case(case, handle(case["input"]))
|
|
passed += int(ok)
|
|
print(f"{case['id']}: {'PASS' if ok else 'FAIL'} ({reason})")
|
|
score = passed / len(cases) if cases else 0.0
|
|
print(f"accuracy={score:.3f} threshold={args.threshold:.3f}")
|
|
return 0 if score >= args.threshold else 1
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|