diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7770b48 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# Copy to .env locally. Never commit real values. +OPENAI_API_KEY= +TAVILY_API_KEY= +VECTOR_DB_URL= +VECTOR_DB_TOKEN= +REGISTRY_URL= +REGISTRY_USER= +REGISTRY_TOKEN= diff --git a/.gitea/issue_templates/bug.md b/.gitea/issue_templates/bug.md new file mode 100644 index 0000000..6ec28ba --- /dev/null +++ b/.gitea/issue_templates/bug.md @@ -0,0 +1,15 @@ +--- +name: Bug report +about: Report a reproducible agent or evaluation problem +title: "[bug] " +labels: bug +--- + +## What happened? + +## Reproduction + +## Expected behavior + +## Evaluation impact + diff --git a/.gitea/issue_templates/feature.md b/.gitea/issue_templates/feature.md new file mode 100644 index 0000000..53984ae --- /dev/null +++ b/.gitea/issue_templates/feature.md @@ -0,0 +1,15 @@ +--- +name: Feature request +about: Propose a scoped capability or prompt change +title: "[feature] " +labels: enhancement +--- + +## Problem + +## Proposed behavior + +## Prompt, tool, or memory impact + +## Acceptance criteria + diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..71f294d --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,27 @@ +name: Deploy + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + validate-image: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + - name: Validate deployment files + run: test -f deploy/Dockerfile && test -f deploy/docker-compose.yml + - name: Build image when registry credentials exist + if: ${{ secrets.REGISTRY_URL != '' && secrets.REGISTRY_TOKEN != '' }} + env: + REGISTRY_URL: ${{ secrets.REGISTRY_URL }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_URL" --username "${{ secrets.REGISTRY_USER }}" --password-stdin + docker build -f deploy/Dockerfile -t "$REGISTRY_URL/agent:${{ gitea.sha }}" . + docker push "$REGISTRY_URL/agent:${{ gitea.sha }}" diff --git a/.gitea/workflows/eval-quality.yml b/.gitea/workflows/eval-quality.yml new file mode 100644 index 0000000..f78e3b2 --- /dev/null +++ b/.gitea/workflows/eval-quality.yml @@ -0,0 +1,19 @@ +name: Eval Gate + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + evaluate: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + - name: Run offline quality gate + run: python3 evals/run_evals.py --threshold 0.95 diff --git a/README.md b/README.md index e6440bd..ff30ef8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,28 @@ -# agent-template +# IT-One AI Agent Template -Canonical public template for autonomous AI agents, prompts, tools, memory, evaluations, deployment, and CI/CD. \ No newline at end of file +A public, reusable starting point for autonomous agents and multi-agent systems. Generate a repository from this template, then rename it using the conventions below. + +## Naming + +- `agent-` — autonomous agent. +- `swarm-` or `orch-` — orchestrator or multi-agent system. +- `eval--benchmark` — dataset or quality benchmark. + +## Layout + +Prompts are versioned separately from implementation. `src/tools` contains integrations, `src/memory` contains retrieval/state adapters, `src/core` contains the agent graph, and `evals` contains deterministic quality checks. + +## Quality gate + +Every pull request runs `.gitea/workflows/eval-quality.yml`. The evaluator must keep Accuracy and Tool Calling Success at or above **95%**. A failing evaluator blocks the protected `main` branch. The sample evaluator is offline-safe; replace its cases and assertions for a real agent. + +## Secrets + +Never commit provider or database credentials. Configure organization-level Actions secrets in Gitea and reference them as `${{ secrets.OPENAI_API_KEY }}`, `${{ secrets.TAVILY_API_KEY }}`, `${{ secrets.VECTOR_DB_URL }}`, and `${{ secrets.REGISTRY_TOKEN }}` only when a project actually needs them. + +## Local development + +```bash +python -m evals.run_evals --threshold 0.95 +python -m src.main +``` diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..ac60b49 --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . . +RUN pip install --no-cache-dir -r requirements.txt +CMD ["python", "-m", "src.main"] diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..0f6bd12 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,8 @@ +services: + agent: + build: + context: .. + dockerfile: deploy/Dockerfile + env_file: + - ../.env + restart: unless-stopped diff --git a/docs/system-architecture.md b/docs/system-architecture.md new file mode 100644 index 0000000..bbd846b --- /dev/null +++ b/docs/system-architecture.md @@ -0,0 +1,14 @@ +# System architecture + +```mermaid +flowchart LR + Input[User request] --> Core[Agent graph] + Core --> Prompt[Versioned prompts] + Core --> Tools[External tools] + Core --> Memory[Memory / RAG] + Core --> Output[Validated response] + Evals[Offline evals] --> Gate[PR quality gate] + Gate --> Core +``` + +The core must keep orchestration deterministic enough to test. Tools should have explicit schemas and timeouts. Memory adapters must make retention and deletion behavior visible. diff --git a/docs/tool-definitions.md b/docs/tool-definitions.md new file mode 100644 index 0000000..a104a0c --- /dev/null +++ b/docs/tool-definitions.md @@ -0,0 +1,8 @@ +# Tool definitions + +Every tool should document its input schema, output schema, timeout, retry policy, authentication source, and failure mode. Production credentials come from Gitea Actions secrets, never from source files. + +| Tool | Purpose | Failure policy | +| --- | --- | --- | +| CRM client | Read or update customer records | Fail closed; redact identifiers | +| Web search | Retrieve public references | Timeout and cite sources | diff --git a/evals/assert_rules.py b/evals/assert_rules.py new file mode 100644 index 0000000..c57de97 --- /dev/null +++ b/evals/assert_rules.py @@ -0,0 +1,8 @@ +def score_case(case: dict, result: dict) -> tuple[bool, str]: + answer = str(result.get("answer", "")) + expected = str(case.get("expected_answer_contains", "")) + if expected and expected.lower() not in answer.lower(): + return False, "answer substring mismatch" + if len(result.get("tool_calls", [])) != int(case.get("expected_tool_calls", 0)): + return False, "tool call count mismatch" + return True, "ok" diff --git a/evals/run_evals.py b/evals/run_evals.py new file mode 100644 index 0000000..03cb9dc --- /dev/null +++ b/evals/run_evals.py @@ -0,0 +1,20 @@ +import argparse, json, pathlib, sys +from src.core.agent import handle +from evals.assert_rules import score_case + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--threshold", type=float, default=0.95) + args = parser.parse_args() + cases = [json.loads(line) for line in pathlib.Path("evals/test_cases.jsonl").read_text().splitlines() if line.strip()] + passed = 0 + for case in cases: + ok, reason = score_case(case, handle(case["input"])) + passed += int(ok) + print(f"{case['id']}: {'PASS' if ok else 'FAIL'} ({reason})") + score = passed / len(cases) if cases else 0.0 + print(f"accuracy={score:.3f} threshold={args.threshold:.3f}") + return 0 if score >= args.threshold else 1 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/test_cases.jsonl b/evals/test_cases.jsonl new file mode 100644 index 0000000..094a063 --- /dev/null +++ b/evals/test_cases.jsonl @@ -0,0 +1,5 @@ +{"id":"greeting","input":"Hello","expected_answer_contains":"Hello","expected_tool_calls":0} +{"id":"empty-safe","input":" ","expected_answer_contains":"","expected_tool_calls":0} +{"id":"explicit-request","input":"List the next step","expected_answer_contains":"List","expected_tool_calls":0} +{"id":"no-invention","input":"Return the supplied value","expected_answer_contains":"supplied","expected_tool_calls":0} +{"id":"traceable","input":"Show the request","expected_answer_contains":"request","expected_tool_calls":0} diff --git a/prompts/few_shots.json b/prompts/few_shots.json new file mode 100644 index 0000000..53801a5 --- /dev/null +++ b/prompts/few_shots.json @@ -0,0 +1,3 @@ +[ + {"user":"Summarize the account status.","assistant":"I will inspect the documented account fields and cite the source of each result."} +] diff --git a/prompts/steps/extract_data.md b/prompts/steps/extract_data.md new file mode 100644 index 0000000..c688d80 --- /dev/null +++ b/prompts/steps/extract_data.md @@ -0,0 +1,3 @@ +# Extract data + +Return a structured object with `facts`, `missing_fields`, and `source_refs`. Keep user-provided values distinct from inferred values. diff --git a/prompts/steps/qualify.md b/prompts/steps/qualify.md new file mode 100644 index 0000000..d843a3b --- /dev/null +++ b/prompts/steps/qualify.md @@ -0,0 +1,6 @@ +# Qualify + +1. Identify the request and desired outcome. +2. Check authorization and irreversible effects. +3. Select only the tools needed. +4. Record assumptions for evaluation. diff --git a/prompts/system.prompt.md b/prompts/system.prompt.md new file mode 100644 index 0000000..ffdf0e4 --- /dev/null +++ b/prompts/system.prompt.md @@ -0,0 +1,3 @@ +# System prompt + +You are a reliable IT-One assistant. Follow the user request, state uncertainty, protect private data, and use tools only when their documented purpose applies. Never invent tool results, credentials, or citations. Ask for clarification when an irreversible action is ambiguous. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2327d12 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "it-one-agent" +version = "0.1.0" +description = "IT-One agent template" +requires-python = ">=3.11" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6c5a5b6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +# Add provider SDKs only when the agent needs them. diff --git a/src/core/agent.py b/src/core/agent.py new file mode 100644 index 0000000..f538b1d --- /dev/null +++ b/src/core/agent.py @@ -0,0 +1,5 @@ +from .state import AgentState + +def handle(request: str) -> dict: + state = AgentState(request=request) + return {"answer": request.strip(), "facts": state.facts, "tool_calls": state.tool_calls} diff --git a/src/core/state.py b/src/core/state.py new file mode 100644 index 0000000..d87c8e1 --- /dev/null +++ b/src/core/state.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass, field + +@dataclass +class AgentState: + request: str + facts: list[str] = field(default_factory=list) + tool_calls: list[str] = field(default_factory=list) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..51ac984 --- /dev/null +++ b/src/main.py @@ -0,0 +1,7 @@ +from src.core.agent import handle + +def main() -> None: + print(handle("offline example")) + +if __name__ == "__main__": + main() diff --git a/src/memory/vector_store.py b/src/memory/vector_store.py new file mode 100644 index 0000000..8cd1807 --- /dev/null +++ b/src/memory/vector_store.py @@ -0,0 +1,8 @@ +"""Memory interface. Production implementations must document retention and deletion.""" + +class VectorStore: + def add(self, key: str, text: str) -> None: + raise NotImplementedError + + def search(self, query: str, limit: int = 5) -> list[dict]: + return [] diff --git a/src/tools/crm_client.py b/src/tools/crm_client.py new file mode 100644 index 0000000..6784a2e --- /dev/null +++ b/src/tools/crm_client.py @@ -0,0 +1,6 @@ +"""Minimal CRM adapter boundary; add a real client behind this interface.""" + +def get_customer(customer_id: str) -> dict: + if not customer_id: + raise ValueError("customer_id is required") + raise NotImplementedError("configure a provider-backed implementation") diff --git a/src/tools/web_search.py b/src/tools/web_search.py new file mode 100644 index 0000000..3699ac8 --- /dev/null +++ b/src/tools/web_search.py @@ -0,0 +1,6 @@ +"""Search adapter boundary with an explicit, testable result shape.""" + +def search(query: str, limit: int = 5) -> list[dict]: + if not query.strip(): + return [] + return [{"title": "offline-placeholder", "url": "about:blank", "snippet": query[:200]}][:limit]