codecortex-context-engine 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codecortex/__init__.py +3 -0
- codecortex/architecture/__init__.py +23 -0
- codecortex/architecture/drift.py +182 -0
- codecortex/architecture/inference.py +160 -0
- codecortex/backends/__init__.py +35 -0
- codecortex/backends/base.py +38 -0
- codecortex/backends/context.py +118 -0
- codecortex/backends/contracts.py +65 -0
- codecortex/backends/factory.py +60 -0
- codecortex/backends/graph.py +107 -0
- codecortex/backends/manager.py +303 -0
- codecortex/backends/mcp_client.py +196 -0
- codecortex/backends/pool.py +128 -0
- codecortex/backends/spec.py +83 -0
- codecortex/backends/symbols.py +189 -0
- codecortex/benchmark.py +211 -0
- codecortex/cli.py +409 -0
- codecortex/config.py +27 -0
- codecortex/context/__init__.py +13 -0
- codecortex/context/budget.py +62 -0
- codecortex/context/integrated.py +60 -0
- codecortex/context/pipeline.py +223 -0
- codecortex/core/__init__.py +1 -0
- codecortex/core/contracts.py +45 -0
- codecortex/core/errors.py +17 -0
- codecortex/core/models.py +69 -0
- codecortex/dashboard.py +259 -0
- codecortex/editing.py +41 -0
- codecortex/engines/__init__.py +5 -0
- codecortex/engines/builtin/__init__.py +5 -0
- codecortex/engines/builtin/factory.py +26 -0
- codecortex/engines/builtin/memory.py +35 -0
- codecortex/engines/builtin/repository.py +73 -0
- codecortex/engines/builtin/symbols.py +89 -0
- codecortex/engines/builtin/validation.py +54 -0
- codecortex/engines/registry.py +26 -0
- codecortex/entrypoint.py +266 -0
- codecortex/evaluation/__init__.py +55 -0
- codecortex/evaluation/external.py +265 -0
- codecortex/evaluation/production.py +670 -0
- codecortex/evaluation/regression.py +188 -0
- codecortex/gateway.py +38 -0
- codecortex/git_intelligence.py +252 -0
- codecortex/indexing/__init__.py +6 -0
- codecortex/indexing/graph.py +78 -0
- codecortex/indexing/impact.py +127 -0
- codecortex/indexing/incremental.py +163 -0
- codecortex/indexing/incremental_graph.py +189 -0
- codecortex/indexing/indexer.py +172 -0
- codecortex/indexing/relationships.py +179 -0
- codecortex/indexing/resolution.py +88 -0
- codecortex/integrations/__init__.py +5 -0
- codecortex/integrations/agents.py +235 -0
- codecortex/interfaces/__init__.py +1 -0
- codecortex/interfaces/mcp_bridge.py +66 -0
- codecortex/languages/__init__.py +5 -0
- codecortex/languages/native.py +166 -0
- codecortex/languages/registry.py +232 -0
- codecortex/mcp/__init__.py +5 -0
- codecortex/mcp/extended.py +114 -0
- codecortex/mcp/server.py +473 -0
- codecortex/memory/__init__.py +11 -0
- codecortex/memory/json_store.py +52 -0
- codecortex/memory/knowledge.py +193 -0
- codecortex/memory/team_store.py +193 -0
- codecortex/orchestrator.py +153 -0
- codecortex/pr_intelligence.py +214 -0
- codecortex/retrieval/__init__.py +16 -0
- codecortex/retrieval/hybrid.py +67 -0
- codecortex/retrieval/index.py +135 -0
- codecortex/retrieval/providers.py +67 -0
- codecortex/retrieval/repository.py +94 -0
- codecortex/router/__init__.py +5 -0
- codecortex/router/router.py +79 -0
- codecortex/runtime.py +69 -0
- codecortex/setup.py +100 -0
- codecortex/symbols/__init__.py +5 -0
- codecortex/symbols/providers.py +192 -0
- codecortex/telemetry/__init__.py +5 -0
- codecortex/telemetry/collector.py +43 -0
- codecortex/tracing/__init__.py +9 -0
- codecortex/tracing/task_trace.py +235 -0
- codecortex/workspace/__init__.py +9 -0
- codecortex/workspace/federation.py +173 -0
- codecortex_context_engine-0.1.0a1.dist-info/METADATA +381 -0
- codecortex_context_engine-0.1.0a1.dist-info/RECORD +90 -0
- codecortex_context_engine-0.1.0a1.dist-info/WHEEL +4 -0
- codecortex_context_engine-0.1.0a1.dist-info/entry_points.txt +3 -0
- codecortex_context_engine-0.1.0a1.dist-info/licenses/LICENSE +201 -0
- codecortex_context_engine-0.1.0a1.dist-info/licenses/NOTICE +2 -0
codecortex/runtime.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Runtime assembly for CLI and integration entry points."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from codecortex.backends.factory import build_backend_stack
|
|
9
|
+
from codecortex.backends.manager import BackendManager
|
|
10
|
+
from codecortex.config import CortexConfig
|
|
11
|
+
from codecortex.engines import EngineRegistry
|
|
12
|
+
from codecortex.gateway import CodeCortexGateway
|
|
13
|
+
from codecortex.memory import JsonMemoryStore
|
|
14
|
+
from codecortex.orchestrator import Orchestrator
|
|
15
|
+
from codecortex.router import AdaptiveRouter
|
|
16
|
+
from codecortex.telemetry import TelemetryCollector
|
|
17
|
+
from codecortex.tracing import TaskTraceRecorder
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(slots=True)
|
|
21
|
+
class CortexRuntime:
|
|
22
|
+
config: CortexConfig
|
|
23
|
+
memory: JsonMemoryStore
|
|
24
|
+
registry: EngineRegistry
|
|
25
|
+
router: AdaptiveRouter
|
|
26
|
+
telemetry: TelemetryCollector
|
|
27
|
+
tracer: TaskTraceRecorder
|
|
28
|
+
backend_manager: BackendManager
|
|
29
|
+
active_backends: tuple[str, ...]
|
|
30
|
+
orchestrator: Orchestrator
|
|
31
|
+
gateway: CodeCortexGateway
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_runtime(project_root: Path | None = None) -> CortexRuntime:
|
|
35
|
+
config = CortexConfig(project_root=(project_root or Path.cwd()).resolve())
|
|
36
|
+
config.ensure_directories()
|
|
37
|
+
memory = JsonMemoryStore(config.memory_dir)
|
|
38
|
+
stack = build_backend_stack(config, memory)
|
|
39
|
+
router = AdaptiveRouter(default_budget=config.default_context_budget)
|
|
40
|
+
telemetry = TelemetryCollector(
|
|
41
|
+
enabled=config.telemetry_enabled,
|
|
42
|
+
log_path=config.state_dir / "runtime" / "events.jsonl",
|
|
43
|
+
)
|
|
44
|
+
tracer = TaskTraceRecorder(config.state_dir / "runtime" / "traces.jsonl")
|
|
45
|
+
orchestrator = Orchestrator(
|
|
46
|
+
registry=stack.registry,
|
|
47
|
+
router=router,
|
|
48
|
+
context_processor=stack.context_processor,
|
|
49
|
+
telemetry=telemetry,
|
|
50
|
+
tracer=tracer,
|
|
51
|
+
)
|
|
52
|
+
gateway = CodeCortexGateway(
|
|
53
|
+
router=router,
|
|
54
|
+
orchestrator=orchestrator,
|
|
55
|
+
registry=stack.registry,
|
|
56
|
+
memory=memory,
|
|
57
|
+
)
|
|
58
|
+
return CortexRuntime(
|
|
59
|
+
config=config,
|
|
60
|
+
memory=memory,
|
|
61
|
+
registry=stack.registry,
|
|
62
|
+
router=router,
|
|
63
|
+
telemetry=telemetry,
|
|
64
|
+
tracer=tracer,
|
|
65
|
+
backend_manager=stack.manager,
|
|
66
|
+
active_backends=stack.active,
|
|
67
|
+
orchestrator=orchestrator,
|
|
68
|
+
gateway=gateway,
|
|
69
|
+
)
|
codecortex/setup.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""One-command project setup and integration discovery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import shutil
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from codecortex.indexing.incremental import IncrementalIndex, IndexStats
|
|
11
|
+
from codecortex.indexing.indexer import ProjectIndexer
|
|
12
|
+
from codecortex.memory.knowledge import ProjectKnowledge, ProjectKnowledgeExtractor
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True, slots=True)
|
|
16
|
+
class SetupResult:
|
|
17
|
+
index: IndexStats
|
|
18
|
+
graph_nodes: int
|
|
19
|
+
graph_edges: int
|
|
20
|
+
symbols: int
|
|
21
|
+
languages: tuple[str, ...]
|
|
22
|
+
knowledge: ProjectKnowledge
|
|
23
|
+
detected_agents: tuple[str, ...]
|
|
24
|
+
integration_file: Path
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ProjectSetup:
|
|
28
|
+
AGENTS = {
|
|
29
|
+
"Claude Code": ("claude",),
|
|
30
|
+
"Codex": ("codex",),
|
|
31
|
+
"OpenCode": ("opencode",),
|
|
32
|
+
"Gemini CLI": ("gemini",),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
def __init__(self, root: Path) -> None:
|
|
36
|
+
self.root = root.resolve()
|
|
37
|
+
self.state = self.root / ".codecortex"
|
|
38
|
+
|
|
39
|
+
def detect_agents(self) -> tuple[str, ...]:
|
|
40
|
+
detected = [
|
|
41
|
+
name
|
|
42
|
+
for name, commands in self.AGENTS.items()
|
|
43
|
+
if any(shutil.which(command) for command in commands)
|
|
44
|
+
]
|
|
45
|
+
if (self.root / ".cursor").exists():
|
|
46
|
+
detected.append("Cursor")
|
|
47
|
+
return tuple(sorted(set(detected)))
|
|
48
|
+
|
|
49
|
+
def _write_config(self) -> None:
|
|
50
|
+
config = {
|
|
51
|
+
"version": 1,
|
|
52
|
+
"context_budget": 32000,
|
|
53
|
+
"hard_context_limit": 128000,
|
|
54
|
+
"telemetry": True,
|
|
55
|
+
}
|
|
56
|
+
path = self.state / "config.json"
|
|
57
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
path.write_text(json.dumps(config, indent=2), encoding="utf-8")
|
|
59
|
+
|
|
60
|
+
def _write_integration(self, detected: tuple[str, ...]) -> Path:
|
|
61
|
+
path = self.state / "integrations" / "mcp.json"
|
|
62
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
payload = {
|
|
64
|
+
"mcpServers": {
|
|
65
|
+
"codecortex": {
|
|
66
|
+
"command": "cortex",
|
|
67
|
+
"args": ["mcp", "--path", str(self.root)],
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
"detectedAgents": list(detected),
|
|
71
|
+
}
|
|
72
|
+
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
73
|
+
return path
|
|
74
|
+
|
|
75
|
+
def run(self) -> SetupResult:
|
|
76
|
+
self._write_config()
|
|
77
|
+
index = IncrementalIndex(self.root).refresh()
|
|
78
|
+
graph = ProjectIndexer(self.root).build()
|
|
79
|
+
graph_path = self.state / "index" / "graph.json"
|
|
80
|
+
graph.save(graph_path)
|
|
81
|
+
knowledge = ProjectKnowledgeExtractor(self.root).extract()
|
|
82
|
+
ProjectKnowledgeExtractor(self.root).save(knowledge)
|
|
83
|
+
detected = self.detect_agents()
|
|
84
|
+
integration = self._write_integration(detected)
|
|
85
|
+
counts = graph.counts()
|
|
86
|
+
symbol_count = sum(
|
|
87
|
+
count
|
|
88
|
+
for kind, count in counts.items()
|
|
89
|
+
if kind not in {"file", "module", "reference"}
|
|
90
|
+
)
|
|
91
|
+
return SetupResult(
|
|
92
|
+
index=index,
|
|
93
|
+
graph_nodes=len(graph.nodes),
|
|
94
|
+
graph_edges=len(graph.edges),
|
|
95
|
+
symbols=symbol_count,
|
|
96
|
+
languages=tuple(name for name, _ in knowledge.languages),
|
|
97
|
+
knowledge=knowledge,
|
|
98
|
+
detected_agents=detected,
|
|
99
|
+
integration_file=integration,
|
|
100
|
+
)
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Lightweight multi-language symbol providers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Protocol
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class SymbolRecord:
|
|
14
|
+
name: str
|
|
15
|
+
kind: str
|
|
16
|
+
path: Path
|
|
17
|
+
line: int
|
|
18
|
+
language: str
|
|
19
|
+
container: str | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SymbolProvider(Protocol):
|
|
23
|
+
language: str
|
|
24
|
+
suffixes: tuple[str, ...]
|
|
25
|
+
|
|
26
|
+
def extract(self, path: Path, source: str) -> list[SymbolRecord]: ...
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class PythonProvider:
|
|
30
|
+
language = "python"
|
|
31
|
+
suffixes = (".py",)
|
|
32
|
+
|
|
33
|
+
def extract(self, path: Path, source: str) -> list[SymbolRecord]:
|
|
34
|
+
try:
|
|
35
|
+
tree = ast.parse(source, filename=str(path))
|
|
36
|
+
except SyntaxError:
|
|
37
|
+
return []
|
|
38
|
+
result: list[SymbolRecord] = []
|
|
39
|
+
parents: dict[ast.AST, str] = {}
|
|
40
|
+
for parent in ast.walk(tree):
|
|
41
|
+
for child in ast.iter_child_nodes(parent):
|
|
42
|
+
if isinstance(parent, ast.ClassDef):
|
|
43
|
+
parents[child] = parent.name
|
|
44
|
+
elif parent in parents:
|
|
45
|
+
parents[child] = parents[parent]
|
|
46
|
+
for node in ast.walk(tree):
|
|
47
|
+
container = parents.get(node)
|
|
48
|
+
if isinstance(node, ast.ClassDef):
|
|
49
|
+
result.append(SymbolRecord(node.name, "class", path, node.lineno, self.language))
|
|
50
|
+
elif isinstance(node, ast.AsyncFunctionDef):
|
|
51
|
+
kind = "method" if container else "async_function"
|
|
52
|
+
result.append(
|
|
53
|
+
SymbolRecord(node.name, kind, path, node.lineno, self.language, container)
|
|
54
|
+
)
|
|
55
|
+
elif isinstance(node, ast.FunctionDef):
|
|
56
|
+
kind = "method" if container else "function"
|
|
57
|
+
result.append(
|
|
58
|
+
SymbolRecord(node.name, kind, path, node.lineno, self.language, container)
|
|
59
|
+
)
|
|
60
|
+
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
61
|
+
names: list[str] = []
|
|
62
|
+
if isinstance(node, ast.Import):
|
|
63
|
+
names = [alias.name for alias in node.names]
|
|
64
|
+
elif node.module:
|
|
65
|
+
names = [node.module]
|
|
66
|
+
for name in names:
|
|
67
|
+
result.append(SymbolRecord(name, "import", path, node.lineno, self.language))
|
|
68
|
+
return result
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True, slots=True)
|
|
72
|
+
class Pattern:
|
|
73
|
+
kind: str
|
|
74
|
+
regex: re.Pattern[str]
|
|
75
|
+
group: int = 1
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class RegexProvider:
|
|
79
|
+
def __init__(self, language: str, suffixes: tuple[str, ...], patterns: list[Pattern]) -> None:
|
|
80
|
+
self.language = language
|
|
81
|
+
self.suffixes = suffixes
|
|
82
|
+
self.patterns = patterns
|
|
83
|
+
|
|
84
|
+
def extract(self, path: Path, source: str) -> list[SymbolRecord]:
|
|
85
|
+
result: list[SymbolRecord] = []
|
|
86
|
+
seen: set[tuple[str, str, int]] = set()
|
|
87
|
+
for pattern in self.patterns:
|
|
88
|
+
for match in pattern.regex.finditer(source):
|
|
89
|
+
name = match.group(pattern.group).strip()
|
|
90
|
+
line = source.count("\n", 0, match.start()) + 1
|
|
91
|
+
key = (name, pattern.kind, line)
|
|
92
|
+
if not name or key in seen:
|
|
93
|
+
continue
|
|
94
|
+
seen.add(key)
|
|
95
|
+
result.append(SymbolRecord(name, pattern.kind, path, line, self.language))
|
|
96
|
+
return result
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _p(kind: str, expression: str, flags: int = re.MULTILINE) -> Pattern:
|
|
100
|
+
return Pattern(kind, re.compile(expression, flags))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_JS_PATTERNS = [
|
|
104
|
+
_p("class", r"^\s*(?:export\s+)?class\s+([A-Za-z_$][\w$]*)"),
|
|
105
|
+
_p("interface", r"^\s*(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)"),
|
|
106
|
+
_p("type", r"^\s*(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\s*="),
|
|
107
|
+
_p("enum", r"^\s*(?:export\s+)?enum\s+([A-Za-z_$][\w$]*)"),
|
|
108
|
+
_p("function", r"^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)"),
|
|
109
|
+
_p("function", r"^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\([^\n]*\)\s*=>"),
|
|
110
|
+
_p("import", r"^\s*import\s+(?:[^\n]+?\s+from\s+)?[\"']([^\"']+)[\"']"),
|
|
111
|
+
_p("export", r"^\s*export\s+\{\s*([^}\n]+)\s*\}"),
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
_GO_PATTERNS = [
|
|
115
|
+
_p("function", r"^\s*func\s+([A-Za-z_]\w*)\s*\("),
|
|
116
|
+
_p("method", r"^\s*func\s*\([^)]*\)\s*([A-Za-z_]\w*)\s*\("),
|
|
117
|
+
_p("type", r"^\s*type\s+([A-Za-z_]\w*)\s+(?:struct|interface)\b"),
|
|
118
|
+
_p("import", r"^\s*import\s+[\"']([^\"']+)[\"']"),
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
_RUST_PATTERNS = [
|
|
122
|
+
_p("function", r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+([A-Za-z_]\w*)"),
|
|
123
|
+
_p("struct", r"^\s*(?:pub\s+)?struct\s+([A-Za-z_]\w*)"),
|
|
124
|
+
_p("enum", r"^\s*(?:pub\s+)?enum\s+([A-Za-z_]\w*)"),
|
|
125
|
+
_p("trait", r"^\s*(?:pub\s+)?trait\s+([A-Za-z_]\w*)"),
|
|
126
|
+
_p("type", r"^\s*(?:pub\s+)?type\s+([A-Za-z_]\w*)"),
|
|
127
|
+
_p("import", r"^\s*use\s+([^;]+);"),
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
_JVM_PATTERNS = [
|
|
131
|
+
_p("class", r"^\s*(?:public\s+|private\s+|protected\s+|abstract\s+|final\s+)*class\s+([A-Za-z_]\w*)"),
|
|
132
|
+
_p("interface", r"^\s*(?:public\s+)?interface\s+([A-Za-z_]\w*)"),
|
|
133
|
+
_p("enum", r"^\s*(?:public\s+)?enum\s+([A-Za-z_]\w*)"),
|
|
134
|
+
_p("import", r"^\s*import\s+([A-Za-z_][\w.*]+)\s*;"),
|
|
135
|
+
_p("method", r"^\s*(?:public|private|protected|static|final|async|virtual|override|synchronized|native|abstract|\s)+\s+[\w<>,.?\[\]]+\s+([A-Za-z_]\w*)\s*\("),
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
_C_PATTERNS = [
|
|
139
|
+
_p("type", r"^\s*(?:typedef\s+)?(?:struct|enum|union)\s+([A-Za-z_]\w*)"),
|
|
140
|
+
_p("class", r"^\s*class\s+([A-Za-z_]\w*)"),
|
|
141
|
+
_p("function", r"^\s*[A-Za-z_][\w\s:*&<>]*\s+([A-Za-z_]\w*)\s*\([^;\n]*\)\s*\{"),
|
|
142
|
+
_p("import", r"^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]"),
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
_PHP_PATTERNS = [
|
|
146
|
+
_p("class", r"^\s*(?:final\s+|abstract\s+)?class\s+([A-Za-z_]\w*)"),
|
|
147
|
+
_p("interface", r"^\s*interface\s+([A-Za-z_]\w*)"),
|
|
148
|
+
_p("trait", r"^\s*trait\s+([A-Za-z_]\w*)"),
|
|
149
|
+
_p("function", r"^\s*(?:public\s+|private\s+|protected\s+|static\s+)*function\s+([A-Za-z_]\w*)"),
|
|
150
|
+
_p("import", r"^\s*use\s+([^;]+);"),
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
_RUBY_PATTERNS = [
|
|
154
|
+
_p("class", r"^\s*class\s+([A-Z]\w*(?:::\w+)*)"),
|
|
155
|
+
_p("module", r"^\s*module\s+([A-Z]\w*(?:::\w+)*)"),
|
|
156
|
+
_p("method", r"^\s*def\s+(?:self\.)?([A-Za-z_]\w*[!?=]?)"),
|
|
157
|
+
_p("import", r"^\s*require(?:_relative)?\s+[\"']([^\"']+)[\"']"),
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class SymbolProviderRegistry:
|
|
162
|
+
def __init__(self) -> None:
|
|
163
|
+
providers: list[SymbolProvider] = [
|
|
164
|
+
PythonProvider(),
|
|
165
|
+
RegexProvider("javascript", (".js", ".jsx", ".mjs", ".cjs"), _JS_PATTERNS),
|
|
166
|
+
RegexProvider("typescript", (".ts", ".tsx", ".mts", ".cts"), _JS_PATTERNS),
|
|
167
|
+
RegexProvider("go", (".go",), _GO_PATTERNS),
|
|
168
|
+
RegexProvider("rust", (".rs",), _RUST_PATTERNS),
|
|
169
|
+
RegexProvider("java", (".java",), _JVM_PATTERNS),
|
|
170
|
+
RegexProvider("csharp", (".cs",), _JVM_PATTERNS),
|
|
171
|
+
RegexProvider("c_cpp", (".c", ".h", ".cc", ".cpp", ".cxx", ".hpp"), _C_PATTERNS),
|
|
172
|
+
RegexProvider("php", (".php",), _PHP_PATTERNS),
|
|
173
|
+
RegexProvider("ruby", (".rb",), _RUBY_PATTERNS),
|
|
174
|
+
]
|
|
175
|
+
self._by_suffix = {
|
|
176
|
+
suffix: provider
|
|
177
|
+
for provider in providers
|
|
178
|
+
for suffix in provider.suffixes
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def suffixes(self) -> frozenset[str]:
|
|
183
|
+
return frozenset(self._by_suffix)
|
|
184
|
+
|
|
185
|
+
def supports(self, path: Path) -> bool:
|
|
186
|
+
return path.suffix.lower() in self._by_suffix
|
|
187
|
+
|
|
188
|
+
def extract(self, path: Path, source: str) -> list[SymbolRecord]:
|
|
189
|
+
provider = self._by_suffix.get(path.suffix.lower())
|
|
190
|
+
if provider is None:
|
|
191
|
+
return []
|
|
192
|
+
return provider.extract(path, source)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Telemetry collection with optional local persistence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from time import time
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(slots=True)
|
|
13
|
+
class TelemetryEvent:
|
|
14
|
+
name: str
|
|
15
|
+
timestamp: float = field(default_factory=time)
|
|
16
|
+
attributes: dict[str, Any] = field(default_factory=dict)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TelemetryCollector:
|
|
20
|
+
def __init__(self, enabled: bool = True, log_path: Path | None = None) -> None:
|
|
21
|
+
self.enabled = enabled
|
|
22
|
+
self.log_path = log_path
|
|
23
|
+
self._events: list[TelemetryEvent] = []
|
|
24
|
+
if self.log_path is not None:
|
|
25
|
+
self.log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
|
|
27
|
+
def emit(self, name: str, **attributes: Any) -> None:
|
|
28
|
+
if not self.enabled:
|
|
29
|
+
return
|
|
30
|
+
event = TelemetryEvent(name=name, attributes=attributes)
|
|
31
|
+
self._events.append(event)
|
|
32
|
+
if self.log_path is not None:
|
|
33
|
+
with self.log_path.open("a", encoding="utf-8") as handle:
|
|
34
|
+
handle.write(json.dumps(asdict(event), ensure_ascii=False, default=str) + "\n")
|
|
35
|
+
|
|
36
|
+
def events(self) -> list[TelemetryEvent]:
|
|
37
|
+
return list(self._events)
|
|
38
|
+
|
|
39
|
+
def snapshot(self) -> dict[str, int]:
|
|
40
|
+
counts: dict[str, int] = {}
|
|
41
|
+
for event in self._events:
|
|
42
|
+
counts[event.name] = counts.get(event.name, 0) + 1
|
|
43
|
+
return counts
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Crash-tolerant task traces for agent workflows and tool execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from collections.abc import AsyncIterator, Iterator
|
|
10
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
11
|
+
from dataclasses import asdict, dataclass
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
_SENSITIVE = re.compile(
|
|
17
|
+
r"^(?:secret|password|passwd|api[_-]?key|authorization|cookie|token)$"
|
|
18
|
+
r"|(?:^|[_-])(?:access|refresh|auth)[_-]?token(?:$|[_-])",
|
|
19
|
+
re.I,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class SpanRecord:
|
|
25
|
+
trace_id: str
|
|
26
|
+
span_id: str
|
|
27
|
+
parent_id: str | None
|
|
28
|
+
name: str
|
|
29
|
+
status: str
|
|
30
|
+
started_at: str
|
|
31
|
+
ended_at: str
|
|
32
|
+
duration_ms: float
|
|
33
|
+
attributes: dict[str, Any]
|
|
34
|
+
error: str | None = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True, slots=True)
|
|
38
|
+
class TraceSummary:
|
|
39
|
+
trace_id: str
|
|
40
|
+
spans: int
|
|
41
|
+
errors: int
|
|
42
|
+
duration_ms: float
|
|
43
|
+
tool_calls: int
|
|
44
|
+
context_tokens: int
|
|
45
|
+
names: tuple[str, ...]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TaskTraceRecorder:
|
|
49
|
+
VERSION = 1
|
|
50
|
+
|
|
51
|
+
def __init__(self, path: Path, max_attribute_chars: int = 2_000) -> None:
|
|
52
|
+
self.path = path
|
|
53
|
+
self.max_attribute_chars = max_attribute_chars
|
|
54
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
|
|
56
|
+
def new_trace_id(self) -> str:
|
|
57
|
+
return uuid.uuid4().hex
|
|
58
|
+
|
|
59
|
+
@contextmanager
|
|
60
|
+
def span(
|
|
61
|
+
self,
|
|
62
|
+
name: str,
|
|
63
|
+
*,
|
|
64
|
+
trace_id: str | None = None,
|
|
65
|
+
parent_id: str | None = None,
|
|
66
|
+
attributes: dict[str, Any] | None = None,
|
|
67
|
+
) -> Iterator[str]:
|
|
68
|
+
resolved_trace = trace_id or self.new_trace_id()
|
|
69
|
+
span_id = uuid.uuid4().hex
|
|
70
|
+
started_wall = datetime.now(UTC).isoformat()
|
|
71
|
+
started = time.perf_counter()
|
|
72
|
+
error: str | None = None
|
|
73
|
+
status = "ok"
|
|
74
|
+
try:
|
|
75
|
+
yield span_id
|
|
76
|
+
except Exception as exc:
|
|
77
|
+
status = "error"
|
|
78
|
+
error = f"{type(exc).__name__}: {exc}"[:500]
|
|
79
|
+
raise
|
|
80
|
+
finally:
|
|
81
|
+
self._append(
|
|
82
|
+
SpanRecord(
|
|
83
|
+
trace_id=resolved_trace,
|
|
84
|
+
span_id=span_id,
|
|
85
|
+
parent_id=parent_id,
|
|
86
|
+
name=name,
|
|
87
|
+
status=status,
|
|
88
|
+
started_at=started_wall,
|
|
89
|
+
ended_at=datetime.now(UTC).isoformat(),
|
|
90
|
+
duration_ms=(time.perf_counter() - started) * 1000,
|
|
91
|
+
attributes=self._sanitize(attributes or {}),
|
|
92
|
+
error=error,
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
@asynccontextmanager
|
|
97
|
+
async def async_span(
|
|
98
|
+
self,
|
|
99
|
+
name: str,
|
|
100
|
+
*,
|
|
101
|
+
trace_id: str | None = None,
|
|
102
|
+
parent_id: str | None = None,
|
|
103
|
+
attributes: dict[str, Any] | None = None,
|
|
104
|
+
) -> AsyncIterator[str]:
|
|
105
|
+
resolved_trace = trace_id or self.new_trace_id()
|
|
106
|
+
span_id = uuid.uuid4().hex
|
|
107
|
+
started_wall = datetime.now(UTC).isoformat()
|
|
108
|
+
started = time.perf_counter()
|
|
109
|
+
error: str | None = None
|
|
110
|
+
status = "ok"
|
|
111
|
+
try:
|
|
112
|
+
yield span_id
|
|
113
|
+
except Exception as exc:
|
|
114
|
+
status = "error"
|
|
115
|
+
error = f"{type(exc).__name__}: {exc}"[:500]
|
|
116
|
+
raise
|
|
117
|
+
finally:
|
|
118
|
+
self._append(
|
|
119
|
+
SpanRecord(
|
|
120
|
+
trace_id=resolved_trace,
|
|
121
|
+
span_id=span_id,
|
|
122
|
+
parent_id=parent_id,
|
|
123
|
+
name=name,
|
|
124
|
+
status=status,
|
|
125
|
+
started_at=started_wall,
|
|
126
|
+
ended_at=datetime.now(UTC).isoformat(),
|
|
127
|
+
duration_ms=(time.perf_counter() - started) * 1000,
|
|
128
|
+
attributes=self._sanitize(attributes or {}),
|
|
129
|
+
error=error,
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
def record(
|
|
134
|
+
self,
|
|
135
|
+
name: str,
|
|
136
|
+
*,
|
|
137
|
+
trace_id: str,
|
|
138
|
+
attributes: dict[str, Any] | None = None,
|
|
139
|
+
parent_id: str | None = None,
|
|
140
|
+
) -> str:
|
|
141
|
+
now = datetime.now(UTC).isoformat()
|
|
142
|
+
span_id = uuid.uuid4().hex
|
|
143
|
+
self._append(
|
|
144
|
+
SpanRecord(
|
|
145
|
+
trace_id=trace_id,
|
|
146
|
+
span_id=span_id,
|
|
147
|
+
parent_id=parent_id,
|
|
148
|
+
name=name,
|
|
149
|
+
status="ok",
|
|
150
|
+
started_at=now,
|
|
151
|
+
ended_at=now,
|
|
152
|
+
duration_ms=0.0,
|
|
153
|
+
attributes=self._sanitize(attributes or {}),
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
return span_id
|
|
157
|
+
|
|
158
|
+
def read(self, trace_id: str | None = None, limit: int = 10_000) -> list[SpanRecord]:
|
|
159
|
+
try:
|
|
160
|
+
lines = self.path.read_text(encoding="utf-8").splitlines()
|
|
161
|
+
except OSError:
|
|
162
|
+
return []
|
|
163
|
+
records: list[SpanRecord] = []
|
|
164
|
+
for line in lines[-max(1, limit) :]:
|
|
165
|
+
try:
|
|
166
|
+
payload = json.loads(line)
|
|
167
|
+
if int(payload.get("version", -1)) != self.VERSION:
|
|
168
|
+
continue
|
|
169
|
+
data = payload["span"]
|
|
170
|
+
record = SpanRecord(
|
|
171
|
+
trace_id=str(data["trace_id"]),
|
|
172
|
+
span_id=str(data["span_id"]),
|
|
173
|
+
parent_id=str(data["parent_id"]) if data.get("parent_id") else None,
|
|
174
|
+
name=str(data["name"]),
|
|
175
|
+
status=str(data["status"]),
|
|
176
|
+
started_at=str(data["started_at"]),
|
|
177
|
+
ended_at=str(data["ended_at"]),
|
|
178
|
+
duration_ms=float(data["duration_ms"]),
|
|
179
|
+
attributes=dict(data.get("attributes", {})),
|
|
180
|
+
error=str(data["error"]) if data.get("error") else None,
|
|
181
|
+
)
|
|
182
|
+
except (json.JSONDecodeError, KeyError, TypeError, ValueError):
|
|
183
|
+
continue
|
|
184
|
+
if trace_id is None or record.trace_id == trace_id:
|
|
185
|
+
records.append(record)
|
|
186
|
+
return records
|
|
187
|
+
|
|
188
|
+
def summarize(self, trace_id: str) -> TraceSummary:
|
|
189
|
+
records = self.read(trace_id)
|
|
190
|
+
return TraceSummary(
|
|
191
|
+
trace_id=trace_id,
|
|
192
|
+
spans=len(records),
|
|
193
|
+
errors=sum(record.status == "error" for record in records),
|
|
194
|
+
duration_ms=sum(record.duration_ms for record in records),
|
|
195
|
+
tool_calls=sum(record.name.startswith("tool.") for record in records),
|
|
196
|
+
context_tokens=sum(
|
|
197
|
+
self._integer_metric(record.attributes.get("context_tokens"))
|
|
198
|
+
for record in records
|
|
199
|
+
),
|
|
200
|
+
names=tuple(record.name for record in records),
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def _append(self, record: SpanRecord) -> None:
|
|
204
|
+
payload = {"version": self.VERSION, "span": asdict(record)}
|
|
205
|
+
with self.path.open("a", encoding="utf-8") as handle:
|
|
206
|
+
handle.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
|
|
207
|
+
handle.write("\n")
|
|
208
|
+
handle.flush()
|
|
209
|
+
|
|
210
|
+
def _sanitize(self, attributes: dict[str, Any]) -> dict[str, Any]:
|
|
211
|
+
sanitized: dict[str, Any] = {}
|
|
212
|
+
for key, value in attributes.items():
|
|
213
|
+
if _SENSITIVE.search(str(key)):
|
|
214
|
+
sanitized[str(key)] = "[REDACTED]"
|
|
215
|
+
continue
|
|
216
|
+
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
217
|
+
text = value if not isinstance(value, str) else value[: self.max_attribute_chars]
|
|
218
|
+
sanitized[str(key)] = text
|
|
219
|
+
else:
|
|
220
|
+
serialized = json.dumps(value, ensure_ascii=False, default=str)
|
|
221
|
+
sanitized[str(key)] = serialized[: self.max_attribute_chars]
|
|
222
|
+
return sanitized
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def _integer_metric(value: Any) -> int:
|
|
226
|
+
if isinstance(value, bool):
|
|
227
|
+
return int(value)
|
|
228
|
+
if isinstance(value, (int, float)):
|
|
229
|
+
return int(value)
|
|
230
|
+
if isinstance(value, str):
|
|
231
|
+
try:
|
|
232
|
+
return int(float(value))
|
|
233
|
+
except ValueError:
|
|
234
|
+
return 0
|
|
235
|
+
return 0
|