fastaiagent 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastaiagent-0.1.0a1/.claude/settings.local.json +58 -0
- fastaiagent-0.1.0a1/.github/ISSUE_TEMPLATE/bug_report.yml +103 -0
- fastaiagent-0.1.0a1/.github/ISSUE_TEMPLATE/config.yml +8 -0
- fastaiagent-0.1.0a1/.github/ISSUE_TEMPLATE/docs_improvement.yml +40 -0
- fastaiagent-0.1.0a1/.github/ISSUE_TEMPLATE/feature_request.yml +41 -0
- fastaiagent-0.1.0a1/.github/ISSUE_TEMPLATE/new_integration.yml +43 -0
- fastaiagent-0.1.0a1/.github/PULL_REQUEST_TEMPLATE.md +23 -0
- fastaiagent-0.1.0a1/.github/labeler.yml +42 -0
- fastaiagent-0.1.0a1/.github/workflows/auto-label.yml +18 -0
- fastaiagent-0.1.0a1/.github/workflows/ci.yml +71 -0
- fastaiagent-0.1.0a1/.github/workflows/docs.yml +31 -0
- fastaiagent-0.1.0a1/.github/workflows/publish.yml +22 -0
- fastaiagent-0.1.0a1/.github/workflows/welcome.yml +35 -0
- fastaiagent-0.1.0a1/.gitignore +50 -0
- fastaiagent-0.1.0a1/.python-version +1 -0
- fastaiagent-0.1.0a1/BENCHMARKS.md +70 -0
- fastaiagent-0.1.0a1/CHANGELOG.md +24 -0
- fastaiagent-0.1.0a1/CODE_OF_CONDUCT.md +38 -0
- fastaiagent-0.1.0a1/CONTRIBUTING.md +172 -0
- fastaiagent-0.1.0a1/LICENSE +201 -0
- fastaiagent-0.1.0a1/PKG-INFO +199 -0
- fastaiagent-0.1.0a1/README.md +137 -0
- fastaiagent-0.1.0a1/benchmarks/bench_checkpoint_overhead.py +36 -0
- fastaiagent-0.1.0a1/benchmarks/bench_cycle_performance.py +32 -0
- fastaiagent-0.1.0a1/benchmarks/bench_eval.py +36 -0
- fastaiagent-0.1.0a1/benchmarks/bench_local_kb.py +34 -0
- fastaiagent-0.1.0a1/benchmarks/bench_replay_load.py +44 -0
- fastaiagent-0.1.0a1/benchmarks/bench_trace_overhead.py +45 -0
- fastaiagent-0.1.0a1/docs/agents/index.md +264 -0
- fastaiagent-0.1.0a1/docs/agents/memory.md +71 -0
- fastaiagent-0.1.0a1/docs/agents/teams.md +83 -0
- fastaiagent-0.1.0a1/docs/agents/tools.md +110 -0
- fastaiagent-0.1.0a1/docs/api-reference/index.md +39 -0
- fastaiagent-0.1.0a1/docs/chains/checkpointing.md +85 -0
- fastaiagent-0.1.0a1/docs/chains/cyclic-workflows.md +75 -0
- fastaiagent-0.1.0a1/docs/chains/hitl.md +93 -0
- fastaiagent-0.1.0a1/docs/chains/index.md +301 -0
- fastaiagent-0.1.0a1/docs/cli/index.md +128 -0
- fastaiagent-0.1.0a1/docs/evaluation/index.md +362 -0
- fastaiagent-0.1.0a1/docs/evaluation/llm-judge.md +80 -0
- fastaiagent-0.1.0a1/docs/evaluation/session-scoring.md +71 -0
- fastaiagent-0.1.0a1/docs/evaluation/trajectory-scoring.md +95 -0
- fastaiagent-0.1.0a1/docs/getting-started/first-agent.md +66 -0
- fastaiagent-0.1.0a1/docs/getting-started/index.md +255 -0
- fastaiagent-0.1.0a1/docs/getting-started/installation.md +57 -0
- fastaiagent-0.1.0a1/docs/guardrails/index.md +362 -0
- fastaiagent-0.1.0a1/docs/index.md +69 -0
- fastaiagent-0.1.0a1/docs/integrations/index.md +280 -0
- fastaiagent-0.1.0a1/docs/knowledge-base/index.md +356 -0
- fastaiagent-0.1.0a1/docs/migration-guides/from-langfuse.md +77 -0
- fastaiagent-0.1.0a1/docs/migration-guides/from-langsmith.md +93 -0
- fastaiagent-0.1.0a1/docs/migration-guides/from-mlflow.md +78 -0
- fastaiagent-0.1.0a1/docs/migration-guides/index.md +7 -0
- fastaiagent-0.1.0a1/docs/platform/index.md +400 -0
- fastaiagent-0.1.0a1/docs/prompts/index.md +324 -0
- fastaiagent-0.1.0a1/docs/replay/index.md +255 -0
- fastaiagent-0.1.0a1/docs/tools/function-tools.md +116 -0
- fastaiagent-0.1.0a1/docs/tools/index.md +130 -0
- fastaiagent-0.1.0a1/docs/tools/mcp-tools.md +46 -0
- fastaiagent-0.1.0a1/docs/tools/rest-tools.md +63 -0
- fastaiagent-0.1.0a1/docs/tools/schema-drift.md +59 -0
- fastaiagent-0.1.0a1/docs/tracing/index.md +293 -0
- fastaiagent-0.1.0a1/docs/tutorials/debug-with-replay.md +77 -0
- fastaiagent-0.1.0a1/docs/tutorials/first-agent.md +54 -0
- fastaiagent-0.1.0a1/docs/tutorials/index.md +13 -0
- fastaiagent-0.1.0a1/docs/tutorials/trace-langchain.md +65 -0
- fastaiagent-0.1.0a1/examples/01_simple_agent.py +33 -0
- fastaiagent-0.1.0a1/examples/02_chain_with_cycles.py +56 -0
- fastaiagent-0.1.0a1/examples/03_guardrails.py +36 -0
- fastaiagent-0.1.0a1/examples/04_agent_replay.py +71 -0
- fastaiagent-0.1.0a1/examples/05_prompt_fragments.py +52 -0
- fastaiagent-0.1.0a1/examples/06_rag_agent.py +49 -0
- fastaiagent-0.1.0a1/examples/07_eval_pipeline.py +52 -0
- fastaiagent-0.1.0a1/examples/08_trace_langchain.py +28 -0
- fastaiagent-0.1.0a1/examples/09_otel_export.py +31 -0
- fastaiagent-0.1.0a1/examples/10_platform_sync.py +46 -0
- fastaiagent-0.1.0a1/examples/11_cli_usage.sh +72 -0
- fastaiagent-0.1.0a1/fastaiagent/__init__.py +70 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/async_utils.py +28 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/config.py +54 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/errors.py +176 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/serialization.py +46 -0
- fastaiagent-0.1.0a1/fastaiagent/_internal/storage.py +92 -0
- fastaiagent-0.1.0a1/fastaiagent/_platform/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/_platform/api.py +112 -0
- fastaiagent-0.1.0a1/fastaiagent/_platform/cache.py +76 -0
- fastaiagent-0.1.0a1/fastaiagent/_version.py +1 -0
- fastaiagent-0.1.0a1/fastaiagent/agent/__init__.py +14 -0
- fastaiagent-0.1.0a1/fastaiagent/agent/agent.py +158 -0
- fastaiagent-0.1.0a1/fastaiagent/agent/executor.py +89 -0
- fastaiagent-0.1.0a1/fastaiagent/agent/memory.py +61 -0
- fastaiagent-0.1.0a1/fastaiagent/agent/team.py +111 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/__init__.py +14 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/chain.py +216 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/checkpoint.py +154 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/executor.py +327 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/node.py +104 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/state.py +60 -0
- fastaiagent-0.1.0a1/fastaiagent/chain/validator.py +72 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/eval.py +27 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/kb.py +32 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/main.py +67 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/prompts.py +46 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/replay.py +31 -0
- fastaiagent-0.1.0a1/fastaiagent/cli/traces.py +54 -0
- fastaiagent-0.1.0a1/fastaiagent/client.py +56 -0
- fastaiagent-0.1.0a1/fastaiagent/deploy/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/deploy/push.py +172 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/__init__.py +16 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/builtins.py +122 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/dataset.py +58 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/evaluate.py +112 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/llm_judge.py +79 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/results.py +52 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/scorer.py +62 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/session.py +48 -0
- fastaiagent-0.1.0a1/fastaiagent/eval/trajectory.py +109 -0
- fastaiagent-0.1.0a1/fastaiagent/guardrail/__init__.py +29 -0
- fastaiagent-0.1.0a1/fastaiagent/guardrail/builtins.py +161 -0
- fastaiagent-0.1.0a1/fastaiagent/guardrail/executor.py +57 -0
- fastaiagent-0.1.0a1/fastaiagent/guardrail/guardrail.py +103 -0
- fastaiagent-0.1.0a1/fastaiagent/guardrail/implementations.py +176 -0
- fastaiagent-0.1.0a1/fastaiagent/integrations/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/integrations/anthropic.py +63 -0
- fastaiagent-0.1.0a1/fastaiagent/integrations/crewai.py +25 -0
- fastaiagent-0.1.0a1/fastaiagent/integrations/langchain.py +72 -0
- fastaiagent-0.1.0a1/fastaiagent/integrations/openai.py +65 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/__init__.py +5 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/chunking.py +101 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/document.py +59 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/embedding.py +78 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/local.py +117 -0
- fastaiagent-0.1.0a1/fastaiagent/kb/search.py +46 -0
- fastaiagent-0.1.0a1/fastaiagent/llm/__init__.py +24 -0
- fastaiagent-0.1.0a1/fastaiagent/llm/client.py +470 -0
- fastaiagent-0.1.0a1/fastaiagent/llm/message.py +115 -0
- fastaiagent-0.1.0a1/fastaiagent/llm/providers/__init__.py +0 -0
- fastaiagent-0.1.0a1/fastaiagent/prompt/__init__.py +7 -0
- fastaiagent-0.1.0a1/fastaiagent/prompt/fragment.py +25 -0
- fastaiagent-0.1.0a1/fastaiagent/prompt/prompt.py +45 -0
- fastaiagent-0.1.0a1/fastaiagent/prompt/registry.py +118 -0
- fastaiagent-0.1.0a1/fastaiagent/prompt/storage.py +97 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/__init__.py +15 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/base.py +96 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/function.py +132 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/mcp.py +116 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/rest.py +95 -0
- fastaiagent-0.1.0a1/fastaiagent/tool/schema.py +155 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/__init__.py +15 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/export.py +32 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/otel.py +50 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/replay.py +171 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/span.py +79 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/storage.py +272 -0
- fastaiagent-0.1.0a1/fastaiagent/trace/tracer.py +26 -0
- fastaiagent-0.1.0a1/fixtures/VERSION +1 -0
- fastaiagent-0.1.0a1/fixtures/agent_simple.json +34 -0
- fastaiagent-0.1.0a1/fixtures/agent_with_guardrails.json +34 -0
- fastaiagent-0.1.0a1/fixtures/chain_cyclic.json +44 -0
- fastaiagent-0.1.0a1/fixtures/chain_full.json +74 -0
- fastaiagent-0.1.0a1/fixtures/chain_simple.json +30 -0
- fastaiagent-0.1.0a1/fixtures/chain_typed_state.json +36 -0
- fastaiagent-0.1.0a1/fixtures/chain_with_hitl.json +30 -0
- fastaiagent-0.1.0a1/fixtures/chain_with_parallel.json +30 -0
- fastaiagent-0.1.0a1/fixtures/eval_dataset_multiturn.json +28 -0
- fastaiagent-0.1.0a1/fixtures/guardrail_code.json +10 -0
- fastaiagent-0.1.0a1/fixtures/guardrail_llm_judge.json +15 -0
- fastaiagent-0.1.0a1/fixtures/guardrail_regex.json +12 -0
- fastaiagent-0.1.0a1/fixtures/prompt_simple.json +10 -0
- fastaiagent-0.1.0a1/fixtures/prompt_with_fragments.json +10 -0
- fastaiagent-0.1.0a1/fixtures/tool_function.json +16 -0
- fastaiagent-0.1.0a1/fixtures/tool_mcp.json +23 -0
- fastaiagent-0.1.0a1/fixtures/tool_rest.json +25 -0
- fastaiagent-0.1.0a1/fixtures/trace_agent.json +73 -0
- fastaiagent-0.1.0a1/fixtures/trace_chain.json +68 -0
- fastaiagent-0.1.0a1/mkdocs.yml +125 -0
- fastaiagent-0.1.0a1/py.typed +0 -0
- fastaiagent-0.1.0a1/pyproject.toml +89 -0
- fastaiagent-0.1.0a1/tests/__init__.py +0 -0
- fastaiagent-0.1.0a1/tests/conftest.py +63 -0
- fastaiagent-0.1.0a1/tests/test_agent.py +318 -0
- fastaiagent-0.1.0a1/tests/test_canonical_format.py +209 -0
- fastaiagent-0.1.0a1/tests/test_chain.py +302 -0
- fastaiagent-0.1.0a1/tests/test_cli.py +45 -0
- fastaiagent-0.1.0a1/tests/test_edge_cases.py +298 -0
- fastaiagent-0.1.0a1/tests/test_eval.py +179 -0
- fastaiagent-0.1.0a1/tests/test_guardrail.py +345 -0
- fastaiagent-0.1.0a1/tests/test_import.py +13 -0
- fastaiagent-0.1.0a1/tests/test_integration_live.py +204 -0
- fastaiagent-0.1.0a1/tests/test_integrations.py +60 -0
- fastaiagent-0.1.0a1/tests/test_internal.py +207 -0
- fastaiagent-0.1.0a1/tests/test_kb.py +135 -0
- fastaiagent-0.1.0a1/tests/test_llm.py +290 -0
- fastaiagent-0.1.0a1/tests/test_platform_client.py +275 -0
- fastaiagent-0.1.0a1/tests/test_prompt.py +103 -0
- fastaiagent-0.1.0a1/tests/test_replay.py +122 -0
- fastaiagent-0.1.0a1/tests/test_tool.py +228 -0
- fastaiagent-0.1.0a1/tests/test_trace.py +177 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"permissions": {
|
|
3
|
+
"allow": [
|
|
4
|
+
"Bash(curl -s \"https://pypi.org/pypi/langgraph-checkpoint/json\")",
|
|
5
|
+
"Bash(curl -s \"https://pypi.org/pypi/langgraph-checkpoint-sqlite/json\")",
|
|
6
|
+
"Bash(curl -s \"https://pypi.org/pypi/langgraph-checkpoint-postgres/json\")",
|
|
7
|
+
"Bash(grep -l \"parallel\\\\|hitl\\\\|checkpoint\" /Users/upendrabhandari/fastaiagent-sdk/fastaiagent/chain/*.py)",
|
|
8
|
+
"WebSearch",
|
|
9
|
+
"WebFetch(domain:www.langchain.com)",
|
|
10
|
+
"Bash(python -c \"import fastaiagent; print\\(fastaiagent.__file__\\)\")",
|
|
11
|
+
"Bash(.venv/bin/python -c \"import fastaiagent; print\\('SDK available'\\)\")",
|
|
12
|
+
"Bash(.venv/bin/python -c ':*)",
|
|
13
|
+
"Bash(ls /Users/upendrabhandari/fastaiagent-sdk/docs/05*)",
|
|
14
|
+
"Bash(ls /Users/upendrabhandari/fastaiagent-sdk/docs/06*)",
|
|
15
|
+
"Bash(ls /Users/upendrabhandari/fastaiagent-sdk/docs/07*)",
|
|
16
|
+
"Bash(ls /Users/upendrabhandari/fastaiagent-sdk/docs/08*)",
|
|
17
|
+
"Bash(ls /Users/upendrabhandari/fastaiagent-sdk/docs/09*)",
|
|
18
|
+
"Bash(xargs ls:*)",
|
|
19
|
+
"Bash(wc -l /Users/upendrabhandari/fastaiagent-sdk/docs/*.md)",
|
|
20
|
+
"Bash(grep -c \"def test_\" tests/*.py)",
|
|
21
|
+
"Read(//Users/upendrabhandari/fastaiagent-sdk/**)",
|
|
22
|
+
"Bash(grep -c \"def test_\" /Users/upendrabhandari/fastaiagent-sdk/tests/*.py)",
|
|
23
|
+
"Bash(echo \"exit: $?\")",
|
|
24
|
+
"Bash(python -m pytest tests/ --collect-only -q)",
|
|
25
|
+
"Bash(python -m pytest tests/ -x -q)",
|
|
26
|
+
"Bash(python -m mypy fastaiagent/ --ignore-missing-imports)",
|
|
27
|
+
"Bash(python benchmarks/bench_trace_overhead.py)",
|
|
28
|
+
"Bash(python benchmarks/bench_checkpoint_overhead.py)",
|
|
29
|
+
"Bash(python benchmarks/bench_cycle_performance.py)",
|
|
30
|
+
"Bash(python benchmarks/bench_replay_load.py)",
|
|
31
|
+
"Bash(python benchmarks/bench_local_kb.py)",
|
|
32
|
+
"Bash(python benchmarks/bench_eval.py)",
|
|
33
|
+
"Bash(python -m pytest tests/test_edge_cases.py tests/test_integrations.py -v --tb=short)",
|
|
34
|
+
"Bash(python -c \"from fastaiagent.prompt.prompt import Prompt; p = Prompt\\(name='x', template='Hello World!', version=1\\); print\\(dir\\(p\\)\\)\")",
|
|
35
|
+
"Bash(python -c \"from fastaiagent.chain import Chain; print\\([m for m in dir\\(Chain\\) if not m.startswith\\('_'\\)]\\)\")",
|
|
36
|
+
"Bash(python -c \"from fastaiagent.guardrail import Guardrail; print\\([m for m in dir\\(Guardrail\\) if not m.startswith\\('_'\\)]\\)\")",
|
|
37
|
+
"Bash(python -c \"from fastaiagent.prompt.prompt import Prompt; p = Prompt\\(name='x', template='Hello World!', version=1\\); print\\(p.render\\(\\)\\)\")",
|
|
38
|
+
"Bash(python -m pytest tests/ -q)",
|
|
39
|
+
"Bash(ruff check:*)",
|
|
40
|
+
"Bash([ ! -f \"docs/$d/index.md\" ])",
|
|
41
|
+
"Bash(python -m mypy fastaiagent/chain/chain.py --ignore-missing-imports)",
|
|
42
|
+
"Bash(ruff format:*)",
|
|
43
|
+
"Bash(python -m pytest tests/ -v --tb=no)",
|
|
44
|
+
"Bash(OPENAI_API_KEY=\"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\" python -m pytest tests/test_integration_live.py -v --tb=short)",
|
|
45
|
+
"Bash(python -c \"from fastaiagent.agent import AgentResult; print\\([a for a in dir\\(AgentResult\\) if not a.startswith\\('_'\\)]\\)\")",
|
|
46
|
+
"Bash(python -c \"from fastaiagent.eval.results import EvalResults; print\\([a for a in dir\\(EvalResults\\) if not a.startswith\\('_'\\)]\\)\")",
|
|
47
|
+
"Bash(python -c \"from fastaiagent.guardrail import no_pii; g = no_pii\\(\\); print\\(g.position, g.blocking, g.guardrail_type\\)\")",
|
|
48
|
+
"Bash(python -c \"from fastaiagent.agent import AgentResult; print\\(AgentResult.model_fields.keys\\(\\)\\)\")",
|
|
49
|
+
"Bash(python -c ':*)",
|
|
50
|
+
"Bash(python -m pytest tests/ -q --ignore=tests/test_integration_live.py)",
|
|
51
|
+
"Bash(pip install:*)",
|
|
52
|
+
"Bash(python -m build)"
|
|
53
|
+
],
|
|
54
|
+
"additionalDirectories": [
|
|
55
|
+
"/Users/upendrabhandari/fastaiagent-sdk"
|
|
56
|
+
]
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
name: Bug Report
|
|
2
|
+
description: Report a bug in the FastAIAgent SDK
|
|
3
|
+
labels: ["bug", "triage"]
|
|
4
|
+
body:
|
|
5
|
+
- type: markdown
|
|
6
|
+
attributes:
|
|
7
|
+
value: |
|
|
8
|
+
Thanks for reporting a bug! Please fill out the information below to help us reproduce and fix it.
|
|
9
|
+
|
|
10
|
+
- type: input
|
|
11
|
+
id: version
|
|
12
|
+
attributes:
|
|
13
|
+
label: SDK Version
|
|
14
|
+
description: Output of `pip show fastaiagent | grep Version`
|
|
15
|
+
placeholder: "0.1.0a1"
|
|
16
|
+
validations:
|
|
17
|
+
required: true
|
|
18
|
+
|
|
19
|
+
- type: dropdown
|
|
20
|
+
id: python-version
|
|
21
|
+
attributes:
|
|
22
|
+
label: Python Version
|
|
23
|
+
options:
|
|
24
|
+
- "3.10"
|
|
25
|
+
- "3.11"
|
|
26
|
+
- "3.12"
|
|
27
|
+
- "3.13"
|
|
28
|
+
validations:
|
|
29
|
+
required: true
|
|
30
|
+
|
|
31
|
+
- type: dropdown
|
|
32
|
+
id: os
|
|
33
|
+
attributes:
|
|
34
|
+
label: Operating System
|
|
35
|
+
options:
|
|
36
|
+
- Linux
|
|
37
|
+
- macOS
|
|
38
|
+
- Windows
|
|
39
|
+
validations:
|
|
40
|
+
required: true
|
|
41
|
+
|
|
42
|
+
- type: dropdown
|
|
43
|
+
id: component
|
|
44
|
+
attributes:
|
|
45
|
+
label: Component
|
|
46
|
+
description: Which part of the SDK is affected?
|
|
47
|
+
options:
|
|
48
|
+
- Agent
|
|
49
|
+
- Chain
|
|
50
|
+
- Tool (Function/REST/MCP)
|
|
51
|
+
- LLM Client
|
|
52
|
+
- Guardrail
|
|
53
|
+
- Tracing
|
|
54
|
+
- Agent Replay
|
|
55
|
+
- Prompts
|
|
56
|
+
- Knowledge Base
|
|
57
|
+
- Evaluation
|
|
58
|
+
- CLI
|
|
59
|
+
- Platform Sync
|
|
60
|
+
- Integration (OpenAI/Anthropic/LangChain/CrewAI)
|
|
61
|
+
- Other
|
|
62
|
+
validations:
|
|
63
|
+
required: true
|
|
64
|
+
|
|
65
|
+
- type: textarea
|
|
66
|
+
id: description
|
|
67
|
+
attributes:
|
|
68
|
+
label: Description
|
|
69
|
+
description: A clear description of the bug
|
|
70
|
+
validations:
|
|
71
|
+
required: true
|
|
72
|
+
|
|
73
|
+
- type: textarea
|
|
74
|
+
id: reproduction
|
|
75
|
+
attributes:
|
|
76
|
+
label: Steps to Reproduce
|
|
77
|
+
description: Minimal code to reproduce the issue
|
|
78
|
+
render: python
|
|
79
|
+
validations:
|
|
80
|
+
required: true
|
|
81
|
+
|
|
82
|
+
- type: textarea
|
|
83
|
+
id: expected
|
|
84
|
+
attributes:
|
|
85
|
+
label: Expected Behavior
|
|
86
|
+
description: What you expected to happen
|
|
87
|
+
validations:
|
|
88
|
+
required: true
|
|
89
|
+
|
|
90
|
+
- type: textarea
|
|
91
|
+
id: actual
|
|
92
|
+
attributes:
|
|
93
|
+
label: Actual Behavior
|
|
94
|
+
description: What actually happened (include error messages/tracebacks)
|
|
95
|
+
render: shell
|
|
96
|
+
validations:
|
|
97
|
+
required: true
|
|
98
|
+
|
|
99
|
+
- type: textarea
|
|
100
|
+
id: additional
|
|
101
|
+
attributes:
|
|
102
|
+
label: Additional Context
|
|
103
|
+
description: Any other relevant information (logs, screenshots, etc.)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
blank_issues_enabled: false
|
|
2
|
+
contact_links:
|
|
3
|
+
- name: Questions & Discussions
|
|
4
|
+
url: https://github.com/fastaifoundry/fastaiagent-sdk/discussions
|
|
5
|
+
about: Ask questions, share ideas, and discuss FastAIAgent SDK
|
|
6
|
+
- name: Discord Community
|
|
7
|
+
url: https://discord.gg/fastaiagent
|
|
8
|
+
about: Chat with the community and get help in real-time
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Documentation Improvement
|
|
2
|
+
description: Suggest an improvement to the documentation
|
|
3
|
+
labels: ["documentation"]
|
|
4
|
+
body:
|
|
5
|
+
- type: input
|
|
6
|
+
id: page
|
|
7
|
+
attributes:
|
|
8
|
+
label: Page / Section
|
|
9
|
+
description: Which documentation page or section needs improvement?
|
|
10
|
+
placeholder: "e.g., getting-started/installation.md or Chains > Checkpointing"
|
|
11
|
+
validations:
|
|
12
|
+
required: true
|
|
13
|
+
|
|
14
|
+
- type: dropdown
|
|
15
|
+
id: type
|
|
16
|
+
attributes:
|
|
17
|
+
label: Type of Improvement
|
|
18
|
+
options:
|
|
19
|
+
- Fix incorrect information
|
|
20
|
+
- Add missing information
|
|
21
|
+
- Improve clarity / wording
|
|
22
|
+
- Add code example
|
|
23
|
+
- Fix broken link
|
|
24
|
+
- Other
|
|
25
|
+
validations:
|
|
26
|
+
required: true
|
|
27
|
+
|
|
28
|
+
- type: textarea
|
|
29
|
+
id: description
|
|
30
|
+
attributes:
|
|
31
|
+
label: Description
|
|
32
|
+
description: What needs to be changed and why?
|
|
33
|
+
validations:
|
|
34
|
+
required: true
|
|
35
|
+
|
|
36
|
+
- type: textarea
|
|
37
|
+
id: suggestion
|
|
38
|
+
attributes:
|
|
39
|
+
label: Suggested Improvement
|
|
40
|
+
description: How you think it should read (optional)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Feature Request
|
|
2
|
+
description: Suggest a new feature or improvement
|
|
3
|
+
labels: ["enhancement"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: summary
|
|
7
|
+
attributes:
|
|
8
|
+
label: Feature Summary
|
|
9
|
+
description: A brief description of the feature
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
|
|
13
|
+
- type: textarea
|
|
14
|
+
id: motivation
|
|
15
|
+
attributes:
|
|
16
|
+
label: Motivation / Use Case
|
|
17
|
+
description: Why do you need this? What problem does it solve?
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
|
|
21
|
+
- type: textarea
|
|
22
|
+
id: proposed
|
|
23
|
+
attributes:
|
|
24
|
+
label: Proposed API / Implementation
|
|
25
|
+
description: How you imagine the feature would work (code examples welcome)
|
|
26
|
+
render: python
|
|
27
|
+
|
|
28
|
+
- type: textarea
|
|
29
|
+
id: alternatives
|
|
30
|
+
attributes:
|
|
31
|
+
label: Alternatives Considered
|
|
32
|
+
description: Other approaches you've considered
|
|
33
|
+
|
|
34
|
+
- type: dropdown
|
|
35
|
+
id: willingness
|
|
36
|
+
attributes:
|
|
37
|
+
label: Willingness to Contribute
|
|
38
|
+
options:
|
|
39
|
+
- "I'd like to implement this myself"
|
|
40
|
+
- "I could help with implementation"
|
|
41
|
+
- "I'd prefer the team to implement this"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: New Integration Request
|
|
2
|
+
description: Request auto-tracing support for a new framework
|
|
3
|
+
labels: ["enhancement", "integration"]
|
|
4
|
+
body:
|
|
5
|
+
- type: input
|
|
6
|
+
id: framework
|
|
7
|
+
attributes:
|
|
8
|
+
label: Framework / Library Name
|
|
9
|
+
placeholder: "e.g., DSPy, PydanticAI, AutoGen"
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
|
|
13
|
+
- type: input
|
|
14
|
+
id: framework-version
|
|
15
|
+
attributes:
|
|
16
|
+
label: Framework Version
|
|
17
|
+
placeholder: "e.g., >= 2.0"
|
|
18
|
+
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: description
|
|
21
|
+
attributes:
|
|
22
|
+
label: What Should Be Traced?
|
|
23
|
+
description: What spans/events would be useful to capture?
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
|
|
27
|
+
- type: textarea
|
|
28
|
+
id: api-design
|
|
29
|
+
attributes:
|
|
30
|
+
label: Proposed API
|
|
31
|
+
description: How the integration should work from the user's perspective
|
|
32
|
+
render: python
|
|
33
|
+
value: |
|
|
34
|
+
import fastaiagent
|
|
35
|
+
fastaiagent.integrations.<framework>.enable()
|
|
36
|
+
|
|
37
|
+
# Existing framework code runs as usual, now with tracing
|
|
38
|
+
|
|
39
|
+
- type: input
|
|
40
|
+
id: docs-link
|
|
41
|
+
attributes:
|
|
42
|
+
label: Framework Documentation Link
|
|
43
|
+
placeholder: "https://..."
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
<!-- Brief description of changes -->
|
|
3
|
+
|
|
4
|
+
## Type of Change
|
|
5
|
+
- [ ] Bug fix
|
|
6
|
+
- [ ] New feature
|
|
7
|
+
- [ ] Enhancement
|
|
8
|
+
- [ ] Documentation
|
|
9
|
+
- [ ] Refactoring
|
|
10
|
+
- [ ] Tests
|
|
11
|
+
|
|
12
|
+
## Related Issues
|
|
13
|
+
<!-- Link to related issues: Fixes #123, Relates to #456 -->
|
|
14
|
+
|
|
15
|
+
## Checklist
|
|
16
|
+
- [ ] Tests pass (`pytest tests/ -v`)
|
|
17
|
+
- [ ] Linting passes (`ruff check . && ruff format --check .`)
|
|
18
|
+
- [ ] Type checking passes (`mypy fastaiagent/`)
|
|
19
|
+
- [ ] Documentation updated (if applicable)
|
|
20
|
+
- [ ] Changelog updated (if user-facing change)
|
|
21
|
+
|
|
22
|
+
## Test Plan
|
|
23
|
+
<!-- How to verify these changes work correctly -->
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
documentation:
|
|
2
|
+
- changed-files:
|
|
3
|
+
- any-glob-to-any-file: ["docs/**", "*.md"]
|
|
4
|
+
tests:
|
|
5
|
+
- changed-files:
|
|
6
|
+
- any-glob-to-any-file: ["tests/**"]
|
|
7
|
+
ci:
|
|
8
|
+
- changed-files:
|
|
9
|
+
- any-glob-to-any-file: [".github/**"]
|
|
10
|
+
agents:
|
|
11
|
+
- changed-files:
|
|
12
|
+
- any-glob-to-any-file: ["fastaiagent/agent/**"]
|
|
13
|
+
chains:
|
|
14
|
+
- changed-files:
|
|
15
|
+
- any-glob-to-any-file: ["fastaiagent/chain/**"]
|
|
16
|
+
tools:
|
|
17
|
+
- changed-files:
|
|
18
|
+
- any-glob-to-any-file: ["fastaiagent/tool/**"]
|
|
19
|
+
guardrails:
|
|
20
|
+
- changed-files:
|
|
21
|
+
- any-glob-to-any-file: ["fastaiagent/guardrail/**"]
|
|
22
|
+
tracing:
|
|
23
|
+
- changed-files:
|
|
24
|
+
- any-glob-to-any-file: ["fastaiagent/trace/**"]
|
|
25
|
+
evaluation:
|
|
26
|
+
- changed-files:
|
|
27
|
+
- any-glob-to-any-file: ["fastaiagent/eval/**"]
|
|
28
|
+
knowledge-base:
|
|
29
|
+
- changed-files:
|
|
30
|
+
- any-glob-to-any-file: ["fastaiagent/kb/**"]
|
|
31
|
+
prompts:
|
|
32
|
+
- changed-files:
|
|
33
|
+
- any-glob-to-any-file: ["fastaiagent/prompt/**"]
|
|
34
|
+
integrations:
|
|
35
|
+
- changed-files:
|
|
36
|
+
- any-glob-to-any-file: ["fastaiagent/integrations/**"]
|
|
37
|
+
platform:
|
|
38
|
+
- changed-files:
|
|
39
|
+
- any-glob-to-any-file: ["fastaiagent/_platform/**", "fastaiagent/deploy/**"]
|
|
40
|
+
cli:
|
|
41
|
+
- changed-files:
|
|
42
|
+
- any-glob-to-any-file: ["fastaiagent/cli/**"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name: Auto Label PRs
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, synchronize]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
pull-requests: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
label:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/labeler@v5
|
|
16
|
+
with:
|
|
17
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
18
|
+
configuration-path: .github/labeler.yml
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
branches: [main]
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ${{ matrix.os }}
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
15
|
+
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python }}
|
|
21
|
+
- name: Install dependencies
|
|
22
|
+
run: pip install -e ".[dev]"
|
|
23
|
+
- name: Run tests
|
|
24
|
+
run: pytest tests/ -v --tb=short --cov=fastaiagent --cov-report=xml
|
|
25
|
+
- name: Upload coverage
|
|
26
|
+
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
|
|
27
|
+
uses: codecov/codecov-action@v4
|
|
28
|
+
|
|
29
|
+
lint:
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
- uses: actions/setup-python@v5
|
|
34
|
+
with:
|
|
35
|
+
python-version: "3.12"
|
|
36
|
+
- run: pip install ruff
|
|
37
|
+
- run: ruff check .
|
|
38
|
+
- run: ruff format --check .
|
|
39
|
+
|
|
40
|
+
typecheck:
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
- uses: actions/setup-python@v5
|
|
45
|
+
with:
|
|
46
|
+
python-version: "3.12"
|
|
47
|
+
- name: Install dependencies
|
|
48
|
+
run: pip install -e ".[dev]"
|
|
49
|
+
- name: Run mypy
|
|
50
|
+
run: mypy fastaiagent/ --ignore-missing-imports
|
|
51
|
+
|
|
52
|
+
optional-deps:
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
strategy:
|
|
55
|
+
fail-fast: false
|
|
56
|
+
matrix:
|
|
57
|
+
extras: ["", "openai", "anthropic", "kb", "all"]
|
|
58
|
+
steps:
|
|
59
|
+
- uses: actions/checkout@v4
|
|
60
|
+
- uses: actions/setup-python@v5
|
|
61
|
+
with:
|
|
62
|
+
python-version: "3.12"
|
|
63
|
+
- name: Install with extras
|
|
64
|
+
run: |
|
|
65
|
+
if [ -z "${{ matrix.extras }}" ]; then
|
|
66
|
+
pip install -e ".[dev]"
|
|
67
|
+
else
|
|
68
|
+
pip install -e ".[${{ matrix.extras }},dev]"
|
|
69
|
+
fi
|
|
70
|
+
- name: Run tests
|
|
71
|
+
run: pytest tests/ -v --tb=short
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: Deploy Documentation
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
deploy:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
|
|
22
|
+
- name: Install dependencies
|
|
23
|
+
run: pip install -e ".[docs]"
|
|
24
|
+
|
|
25
|
+
- name: Build docs (PR validation)
|
|
26
|
+
if: github.event_name == 'pull_request'
|
|
27
|
+
run: mkdocs build --strict
|
|
28
|
+
|
|
29
|
+
- name: Deploy docs (main branch)
|
|
30
|
+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
31
|
+
run: mkdocs gh-deploy --force
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
tags: ["v*"]
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
publish:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
environment: pypi
|
|
10
|
+
permissions:
|
|
11
|
+
id-token: write
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
- name: Install build tools
|
|
18
|
+
run: pip install build
|
|
19
|
+
- name: Build package
|
|
20
|
+
run: python -m build
|
|
21
|
+
- name: Publish to PyPI
|
|
22
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
name: Welcome New Contributors
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
issues:
|
|
5
|
+
types: [opened]
|
|
6
|
+
pull_request_target:
|
|
7
|
+
types: [opened]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
issues: write
|
|
11
|
+
pull-requests: write
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
welcome:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/first-interaction@v1
|
|
18
|
+
with:
|
|
19
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
20
|
+
issue-message: |
|
|
21
|
+
Welcome to FastAIAgent SDK! Thanks for opening your first issue.
|
|
22
|
+
|
|
23
|
+
A maintainer will review this shortly. In the meantime:
|
|
24
|
+
- Check the [documentation](https://docs.fastaiagent.net)
|
|
25
|
+
- Browse [discussions](https://github.com/fastaifoundry/fastaiagent-sdk/discussions) for similar topics
|
|
26
|
+
- Join our [Discord](https://discord.gg/fastaiagent) for real-time help
|
|
27
|
+
pr-message: |
|
|
28
|
+
Welcome to FastAIAgent SDK! Thanks for your first pull request.
|
|
29
|
+
|
|
30
|
+
A maintainer will review this shortly. Please make sure:
|
|
31
|
+
- Tests pass: `pytest tests/ -v`
|
|
32
|
+
- Linting passes: `ruff check . && ruff format --check .`
|
|
33
|
+
- Type checks pass: `mypy fastaiagent/`
|
|
34
|
+
|
|
35
|
+
Check the [contributing guide](https://github.com/fastaifoundry/fastaiagent-sdk/blob/main/CONTRIBUTING.md) for details.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
|
|
7
|
+
# Distribution / packaging
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
*.egg
|
|
12
|
+
.eggs/
|
|
13
|
+
|
|
14
|
+
# Virtual environments
|
|
15
|
+
.venv/
|
|
16
|
+
venv/
|
|
17
|
+
ENV/
|
|
18
|
+
|
|
19
|
+
# IDE
|
|
20
|
+
.idea/
|
|
21
|
+
.vscode/
|
|
22
|
+
*.swp
|
|
23
|
+
*.swo
|
|
24
|
+
*~
|
|
25
|
+
|
|
26
|
+
# Testing
|
|
27
|
+
.pytest_cache/
|
|
28
|
+
.coverage
|
|
29
|
+
htmlcov/
|
|
30
|
+
coverage.xml
|
|
31
|
+
|
|
32
|
+
# Type checking
|
|
33
|
+
.mypy_cache/
|
|
34
|
+
|
|
35
|
+
# Linting
|
|
36
|
+
.ruff_cache/
|
|
37
|
+
|
|
38
|
+
# SDK local data
|
|
39
|
+
.fastaiagent/
|
|
40
|
+
.prompts/
|
|
41
|
+
|
|
42
|
+
# MkDocs build output
|
|
43
|
+
site/
|
|
44
|
+
|
|
45
|
+
# Internal / private
|
|
46
|
+
claude_files/
|
|
47
|
+
|
|
48
|
+
# OS
|
|
49
|
+
.DS_Store
|
|
50
|
+
Thumbs.db
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# FastAIAgent SDK — Performance Benchmarks
|
|
2
|
+
|
|
3
|
+
## Summary
|
|
4
|
+
|
|
5
|
+
| Benchmark | Target | Measured | Status |
|
|
6
|
+
|-----------|--------|----------|--------|
|
|
7
|
+
| Trace overhead | < 5% | 19.5% | ⚠️ Above target |
|
|
8
|
+
| Checkpoint per node | < 50ms | 0.1ms | ✅ Pass |
|
|
9
|
+
| Cycle performance (100 iterations) | < 10s | < 0.001s | ✅ Pass |
|
|
10
|
+
| Replay load (1000 spans) | < 2s | 0.001s | ✅ Pass |
|
|
11
|
+
| KB search (1000 chunks) | < 500ms | 9.4ms | ✅ Pass |
|
|
12
|
+
| Eval throughput (100 cases, 2 scorers) | < 60s | 0.001s | ✅ Pass |
|
|
13
|
+
|
|
14
|
+
## Environment
|
|
15
|
+
|
|
16
|
+
- Python 3.12, macOS (Apple Silicon)
|
|
17
|
+
- fastaiagent v0.1.0a1
|
|
18
|
+
- Benchmarks use synthetic data (no real LLM calls)
|
|
19
|
+
|
|
20
|
+
## Known Issues
|
|
21
|
+
|
|
22
|
+
**Trace overhead (19.5%):** The OTel `TracerProvider` + `LocalStorageProcessor` adds
|
|
23
|
+
measurable overhead on high-frequency synthetic calls. In real-world usage with LLM
|
|
24
|
+
latency (100ms–2s per call), trace overhead is negligible (< 0.5%). Optimization
|
|
25
|
+
candidates: batch SQLite writes, reduce span attribute serialization.
|
|
26
|
+
|
|
27
|
+
## How to Reproduce
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install -e ".[dev]"
|
|
31
|
+
|
|
32
|
+
python benchmarks/bench_trace_overhead.py
|
|
33
|
+
python benchmarks/bench_checkpoint_overhead.py
|
|
34
|
+
python benchmarks/bench_cycle_performance.py
|
|
35
|
+
python benchmarks/bench_replay_load.py
|
|
36
|
+
python benchmarks/bench_local_kb.py
|
|
37
|
+
python benchmarks/bench_eval.py
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Benchmark Details
|
|
41
|
+
|
|
42
|
+
### Trace Overhead (`bench_trace_overhead.py`)
|
|
43
|
+
|
|
44
|
+
Measures the cost of OTel tracing on agent execution. Runs 1000 iterations with
|
|
45
|
+
and without tracing, compares median execution time. Target: < 5% overhead.
|
|
46
|
+
|
|
47
|
+
### Checkpoint Overhead (`bench_checkpoint_overhead.py`)
|
|
48
|
+
|
|
49
|
+
Measures SQLite checkpoint write time per chain node. Runs 100 checkpoints and
|
|
50
|
+
reports per-node time. Target: < 50ms per node.
|
|
51
|
+
|
|
52
|
+
### Cycle Performance (`bench_cycle_performance.py`)
|
|
53
|
+
|
|
54
|
+
Measures chain cycle execution with 100 iterations. Reports total time excluding
|
|
55
|
+
LLM calls. Target: < 10s total.
|
|
56
|
+
|
|
57
|
+
### Replay Load (`bench_replay_load.py`)
|
|
58
|
+
|
|
59
|
+
Measures time to load and parse a trace with 1000 spans from SQLite storage.
|
|
60
|
+
Target: < 2s.
|
|
61
|
+
|
|
62
|
+
### Local KB Search (`bench_local_kb.py`)
|
|
63
|
+
|
|
64
|
+
Indexes 1000 chunks with SimpleEmbedder, then searches 10 queries. Reports
|
|
65
|
+
per-query search latency. Target: < 500ms per query.
|
|
66
|
+
|
|
67
|
+
### Eval Throughput (`bench_eval.py`)
|
|
68
|
+
|
|
69
|
+
Evaluates 100 test cases with 2 scorers (exact_match, contains). Reports total
|
|
70
|
+
evaluation time excluding LLM calls. Target: < 60s total.
|