codepulse-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codepulse/__init__.py +5 -0
- codepulse/api/app.py +1336 -0
- codepulse/api/errors.py +172 -0
- codepulse/api/security.py +189 -0
- codepulse/auth.py +406 -0
- codepulse/cli.py +1052 -0
- codepulse/config.py +475 -0
- codepulse/data_location.py +78 -0
- codepulse/embeddings/chunker.py +138 -0
- codepulse/embeddings/embedder.py +120 -0
- codepulse/git_diff.py +201 -0
- codepulse/indexer/engine.py +2655 -0
- codepulse/indexer/ignore.py +120 -0
- codepulse/ingestion_safety.py +169 -0
- codepulse/jobs.py +257 -0
- codepulse/mcp_server.py +269 -0
- codepulse/models.py +140 -0
- codepulse/observability.py +130 -0
- codepulse/parser/__init__.py +8 -0
- codepulse/parser/base.py +58 -0
- codepulse/parser/factory.py +37 -0
- codepulse/parser/graph.py +502 -0
- codepulse/parser/javascript.py +428 -0
- codepulse/parser/module_graph.py +232 -0
- codepulse/parser/python.py +331 -0
- codepulse/rate_limit.py +142 -0
- codepulse/repository_identity.py +83 -0
- codepulse/search/bm25.py +342 -0
- codepulse/search/ranker.py +382 -0
- codepulse/search/tokenizer.py +80 -0
- codepulse/search/vector.py +357 -0
- codepulse/storage/db.py +526 -0
- codepulse/storage/migrations.py +347 -0
- codepulse/storage/schema_migrations/README +1 -0
- codepulse/storage/schema_migrations/env.py +70 -0
- codepulse/storage/schema_migrations/script.py.mako +28 -0
- codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
- codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
- codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
- codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
- codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
- codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
- codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
- codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
- codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
- codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
- codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
- codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
- codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
- codepulse/symbol_identity.py +29 -0
- codepulse_engine-0.1.0.dist-info/METADATA +191 -0
- codepulse_engine-0.1.0.dist-info/RECORD +56 -0
- codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
- codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
- codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
- codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
codepulse/mcp_server.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Phase 5.3 -- MCP server (roadmap §37 item 5.3; §13 MCP Strategy).
|
|
3
|
+
|
|
4
|
+
§13, quoted and held to strictly: *"Recommendation: yes, expose an MCP
|
|
5
|
+
server -- but strictly as an adapter, not a parallel implementation.
|
|
6
|
+
Each MCP tool calls the exact same internal Python functions the REST
|
|
7
|
+
API calls."* *"Proposed tools: search_code, get_symbol, find_callers,
|
|
8
|
+
find_callees, impact_analysis, find_similar, get_context (the §12
|
|
9
|
+
bundle)."* *"Permissions/repository selection: MCP connections are
|
|
10
|
+
scoped to one or more repositories at connection-setup time (via the
|
|
11
|
+
same API-key/permission model as REST -- §16), not per-tool-call."*
|
|
12
|
+
*"Failure behavior: tool calls return structured errors
|
|
13
|
+
({"error": "repository_not_indexed", "message": "..."}) rather than
|
|
14
|
+
throwing opaque exceptions."*
|
|
15
|
+
|
|
16
|
+
Architecture: this module is a THIN adapter over `IndexingEngine` and
|
|
17
|
+
`codepulse.auth`/`codepulse.api.security`'s existing authorization
|
|
18
|
+
primitives -- it introduces no new graph/retrieval/authorization logic
|
|
19
|
+
of its own. `build_mcp_server(storage, engine, api_key)` resolves the
|
|
20
|
+
given API key to an `AuthorizedKey` exactly ONCE, via the same
|
|
21
|
+
`verify_api_key` (Phase 1.1) function REST's own `require_api_key`
|
|
22
|
+
dependency calls -- not a second authentication mechanism -- and every
|
|
23
|
+
tool call authorizes against that SAME resolved `repository_ids` set,
|
|
24
|
+
never re-authenticating per call (per §13's own explicit "not
|
|
25
|
+
per-tool-call" instruction; FastAPI's per-request `Depends(
|
|
26
|
+
require_api_key)` mechanism does not apply here since MCP tool calls
|
|
27
|
+
are not individual HTTP requests).
|
|
28
|
+
|
|
29
|
+
Each tool below calls the exact same `IndexingEngine` method its REST
|
|
30
|
+
counterpart calls (`search`, `analyze_impact`, `find_similar_symbols`,
|
|
31
|
+
`get_context_bundle`) -- `find_callers`/`find_callees` both reuse
|
|
32
|
+
`analyze_impact` internally (there is no dedicated standalone
|
|
33
|
+
REST/engine method for either; §13 names them as separate TOOLS, not
|
|
34
|
+
as evidence of a separate implementation, so they are built as thin
|
|
35
|
+
projections of the one already-tested `analyze_impact` result rather
|
|
36
|
+
than a second call-graph traversal). `get_symbol`'s `called_by`/`calls`
|
|
37
|
+
query mirrors `GET /v1/symbols/{id}`'s own tested query exactly
|
|
38
|
+
(including the Phase 4.1 compatibility correction's `relationship_type
|
|
39
|
+
== "calls"` filter) -- a small, deliberate, disclosed duplication of a
|
|
40
|
+
plain parameterized DB read (not a graph algorithm), not extracted into
|
|
41
|
+
a third shared location this milestone; see CODEPULSE_PROGRESS.md's
|
|
42
|
+
Phase 5.3 entry for the explicit reasoning.
|
|
43
|
+
|
|
44
|
+
Backend-hardening addition (not a roadmap sprint item; §13's own
|
|
45
|
+
literal tool list names exactly 7): `module_structure` -- an 8th tool,
|
|
46
|
+
giving Phase 4.3's `IndexingEngine.analyze_module_structure` (built
|
|
47
|
+
with no MCP caller at the time, by that milestone's own explicit
|
|
48
|
+
approved scope) its first MCP surface, matching the REST/CLI query
|
|
49
|
+
surfaces added in the same pass.
|
|
50
|
+
"""
|
|
51
|
+
from typing import Any, Optional
|
|
52
|
+
|
|
53
|
+
from mcp.server.fastmcp import FastMCP
|
|
54
|
+
|
|
55
|
+
from codepulse.auth import verify_api_key
|
|
56
|
+
from codepulse.api.security import AuthorizedKey
|
|
57
|
+
from codepulse.indexer.engine import IndexingEngine
|
|
58
|
+
from codepulse.storage.db import APIKeyRepositoryModel, CallEdge, StorageManager, SymbolModel
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class MCPAuthenticationError(Exception):
|
|
62
|
+
"""Raised by `build_mcp_server` when the supplied API key is missing, unknown, or revoked."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _structured_error(code: str, message: str) -> dict[str, str]:
|
|
66
|
+
"""§13's own literal failure-behavior contract: `{"error": "...", "message": "..."}`."""
|
|
67
|
+
return {"error": code, "message": message}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def build_mcp_server(
|
|
71
|
+
storage: StorageManager, engine: IndexingEngine, api_key: str, name: str = "codepulse"
|
|
72
|
+
) -> FastMCP:
|
|
73
|
+
"""
|
|
74
|
+
Builds and returns a `FastMCP` server instance scoped to the
|
|
75
|
+
repositories the given `api_key` is authorized for -- resolved
|
|
76
|
+
ONCE, at connection-setup time, per §13's explicit instruction.
|
|
77
|
+
Raises `MCPAuthenticationError` for a missing/unknown/revoked key,
|
|
78
|
+
mirroring `require_api_key`'s own HTTP 401 case (there is no HTTP
|
|
79
|
+
response here to return, so this is a plain exception the caller
|
|
80
|
+
-- e.g. `codepulse serve`, Phase 6 -- decides how to report).
|
|
81
|
+
"""
|
|
82
|
+
api_key_id = verify_api_key(storage, api_key)
|
|
83
|
+
if api_key_id is None:
|
|
84
|
+
raise MCPAuthenticationError("Invalid or revoked API key")
|
|
85
|
+
|
|
86
|
+
session = storage.get_session()
|
|
87
|
+
try:
|
|
88
|
+
repo_ids = {
|
|
89
|
+
row.repo_id
|
|
90
|
+
for row in session.query(APIKeyRepositoryModel)
|
|
91
|
+
.filter(APIKeyRepositoryModel.api_key_id == api_key_id)
|
|
92
|
+
.all()
|
|
93
|
+
}
|
|
94
|
+
finally:
|
|
95
|
+
session.close()
|
|
96
|
+
|
|
97
|
+
authorized = AuthorizedKey(api_key_id=api_key_id, repository_ids=frozenset(repo_ids))
|
|
98
|
+
|
|
99
|
+
mcp: FastMCP = FastMCP(name)
|
|
100
|
+
|
|
101
|
+
def _authorize(repo_id: int) -> Optional[dict[str, str]]:
|
|
102
|
+
if repo_id not in authorized.repository_ids:
|
|
103
|
+
return _structured_error(
|
|
104
|
+
"repository_not_authorized",
|
|
105
|
+
f"This connection is not authorized for repository {repo_id}.",
|
|
106
|
+
)
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
def _load_symbol(symbol_id: int) -> Optional[dict[str, Any]]:
|
|
110
|
+
"""Extracts every field a tool might need into a plain dict
|
|
111
|
+
WHILE the session is open (SQLAlchemy relationship access after
|
|
112
|
+
`session.close()` would raise `DetachedInstanceError`), exactly
|
|
113
|
+
matching the "access `sym.file` before closing" pattern every
|
|
114
|
+
REST symbol route already follows."""
|
|
115
|
+
db_session = storage.get_session()
|
|
116
|
+
try:
|
|
117
|
+
sym = db_session.query(SymbolModel).filter(SymbolModel.id == symbol_id).first()
|
|
118
|
+
if not sym or not sym.file:
|
|
119
|
+
return None
|
|
120
|
+
return {
|
|
121
|
+
"id": sym.id,
|
|
122
|
+
"name": sym.name,
|
|
123
|
+
"type": sym.type,
|
|
124
|
+
"file_path": sym.file.path,
|
|
125
|
+
"repo_id": sym.file.repo_id,
|
|
126
|
+
"signature": sym.signature,
|
|
127
|
+
"docstring": sym.docstring,
|
|
128
|
+
"body": sym.body,
|
|
129
|
+
"complexity": sym.complexity,
|
|
130
|
+
}
|
|
131
|
+
finally:
|
|
132
|
+
db_session.close()
|
|
133
|
+
|
|
134
|
+
@mcp.tool()
|
|
135
|
+
def search_code(
|
|
136
|
+
query: str, repository_id: int, limit: int = 20, search_type: str = "hybrid"
|
|
137
|
+
) -> dict[str, Any]:
|
|
138
|
+
"""Hybrid/keyword/semantic/RRF search over an indexed repository's code, scoped to this connection's authorized repositories."""
|
|
139
|
+
auth_error = _authorize(repository_id)
|
|
140
|
+
if auth_error:
|
|
141
|
+
return auth_error
|
|
142
|
+
results = engine.search(query, limit=limit, search_type=search_type, repository_id=repository_id)
|
|
143
|
+
return {"results": [r.to_dict() for r in results]}
|
|
144
|
+
|
|
145
|
+
@mcp.tool()
|
|
146
|
+
def get_symbol(symbol_id: int) -> dict[str, Any]:
|
|
147
|
+
"""Fetches one symbol's definition plus its direct callers/callees (calls-only, matching GET /v1/symbols/{id})."""
|
|
148
|
+
loaded = _load_symbol(symbol_id)
|
|
149
|
+
if loaded is None:
|
|
150
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
151
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
152
|
+
if auth_error:
|
|
153
|
+
return auth_error
|
|
154
|
+
|
|
155
|
+
db_session = storage.get_session()
|
|
156
|
+
try:
|
|
157
|
+
called_by = (
|
|
158
|
+
db_session.query(CallEdge)
|
|
159
|
+
.filter(CallEdge.target_symbol_name == loaded["name"])
|
|
160
|
+
.filter(CallEdge.repo_id == loaded["repo_id"])
|
|
161
|
+
.filter(CallEdge.relationship_type == "calls")
|
|
162
|
+
.all()
|
|
163
|
+
)
|
|
164
|
+
calls = (
|
|
165
|
+
db_session.query(CallEdge)
|
|
166
|
+
.filter(CallEdge.source_symbol_name == loaded["name"])
|
|
167
|
+
.filter(CallEdge.repo_id == loaded["repo_id"])
|
|
168
|
+
.filter(CallEdge.relationship_type == "calls")
|
|
169
|
+
.all()
|
|
170
|
+
)
|
|
171
|
+
finally:
|
|
172
|
+
db_session.close()
|
|
173
|
+
|
|
174
|
+
return {
|
|
175
|
+
"symbol_id": loaded["id"],
|
|
176
|
+
"name": loaded["name"],
|
|
177
|
+
"type": loaded["type"],
|
|
178
|
+
"file": loaded["file_path"],
|
|
179
|
+
"docstring": loaded["docstring"],
|
|
180
|
+
"source_code": loaded["body"],
|
|
181
|
+
"called_by": [
|
|
182
|
+
{"symbol": d.source_symbol_name, "file": d.source_file, "confidence": d.confidence}
|
|
183
|
+
for d in called_by
|
|
184
|
+
],
|
|
185
|
+
"calls": [
|
|
186
|
+
{"symbol": d.target_symbol_name, "file": d.target_file, "confidence": d.confidence}
|
|
187
|
+
for d in calls
|
|
188
|
+
],
|
|
189
|
+
"complexity": loaded["complexity"],
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
@mcp.tool()
|
|
193
|
+
def find_callers(symbol_id: int, depth: int = 3) -> dict[str, Any]:
|
|
194
|
+
"""Symbols that call this one, direct and transitive, confidence-scored and depth-bucketed (a projection of impact_analysis's own callers side)."""
|
|
195
|
+
loaded = _load_symbol(symbol_id)
|
|
196
|
+
if loaded is None:
|
|
197
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
198
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
199
|
+
if auth_error:
|
|
200
|
+
return auth_error
|
|
201
|
+
impact = engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
|
|
202
|
+
return {"symbol": loaded["name"], "callers": impact["call_chain"]}
|
|
203
|
+
|
|
204
|
+
@mcp.tool()
|
|
205
|
+
def find_callees(symbol_id: int, depth: int = 3) -> dict[str, Any]:
|
|
206
|
+
"""Symbols this one calls, direct and transitive, confidence-scored and depth-bucketed (a projection of impact_analysis's own callees side)."""
|
|
207
|
+
loaded = _load_symbol(symbol_id)
|
|
208
|
+
if loaded is None:
|
|
209
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
210
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
211
|
+
if auth_error:
|
|
212
|
+
return auth_error
|
|
213
|
+
impact = engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
|
|
214
|
+
return {"symbol": loaded["name"], "callees": impact["dependencies"]}
|
|
215
|
+
|
|
216
|
+
@mcp.tool()
|
|
217
|
+
def impact_analysis(symbol_id: int, depth: int = 3) -> dict[str, Any]:
|
|
218
|
+
"""Direct-vs-transitive impact bucketing for a symbol (calls only; see CODEPULSE_PROGRESS.md's Phase 4.2 Decision 1)."""
|
|
219
|
+
loaded = _load_symbol(symbol_id)
|
|
220
|
+
if loaded is None:
|
|
221
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
222
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
223
|
+
if auth_error:
|
|
224
|
+
return auth_error
|
|
225
|
+
return engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
|
|
226
|
+
|
|
227
|
+
@mcp.tool()
|
|
228
|
+
def find_similar(symbol_id: int, threshold: float = 0.75, limit: int = 10) -> dict[str, Any]:
|
|
229
|
+
"""Semantically similar symbols within the SAME repository (embedding-based, repository-scoped)."""
|
|
230
|
+
loaded = _load_symbol(symbol_id)
|
|
231
|
+
if loaded is None:
|
|
232
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
233
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
234
|
+
if auth_error:
|
|
235
|
+
return auth_error
|
|
236
|
+
query_text = f"{loaded['signature']}\n{loaded['docstring'] or ''}\n{loaded['body'][:500]}"
|
|
237
|
+
similar = engine.find_similar_symbols(query_text, loaded["name"], loaded["repo_id"], threshold=threshold, limit=limit)
|
|
238
|
+
return {"query_symbol": {"id": loaded["id"], "name": loaded["name"]}, "similar": similar}
|
|
239
|
+
|
|
240
|
+
@mcp.tool()
|
|
241
|
+
def get_context(symbol_id: int, include_body: bool = False, max_related: int = 10) -> dict[str, Any]:
|
|
242
|
+
"""The §12 agent-native context bundle: definition, direct callers/callees, semantically related symbols, snapshot commit, and a truncation flag."""
|
|
243
|
+
loaded = _load_symbol(symbol_id)
|
|
244
|
+
if loaded is None:
|
|
245
|
+
return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
|
|
246
|
+
auth_error = _authorize(loaded["repo_id"])
|
|
247
|
+
if auth_error:
|
|
248
|
+
return auth_error
|
|
249
|
+
return engine.get_context_bundle(
|
|
250
|
+
loaded["name"], loaded["file_path"], loaded["repo_id"],
|
|
251
|
+
signature=loaded["signature"], docstring=loaded["docstring"], body=loaded["body"],
|
|
252
|
+
include_body=include_body, max_related=max_related,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
@mcp.tool()
|
|
256
|
+
def module_structure(repository_id: int) -> dict[str, Any]:
|
|
257
|
+
"""Module clusters + raw inbound/outbound degree rankings for a repository's file-level import graph (Phase 4.3's ModuleGraphBuilder, reused unmodified)."""
|
|
258
|
+
auth_error = _authorize(repository_id)
|
|
259
|
+
if auth_error:
|
|
260
|
+
return auth_error
|
|
261
|
+
module_graph = engine.analyze_module_structure(repository_id)
|
|
262
|
+
return {
|
|
263
|
+
"repository_id": repository_id,
|
|
264
|
+
"clusters": module_graph.clusters,
|
|
265
|
+
"ranked_by_inbound": module_graph.rank_by_inbound(),
|
|
266
|
+
"ranked_by_outbound": module_graph.rank_by_outbound(),
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return mcp
|
codepulse/models.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Optional
|
|
6
|
+
from pydantic import BaseModel, Field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SymbolType(str, Enum):
|
|
10
|
+
# Backend-hardening cleanup (not a roadmap sprint item): INTERFACE and
|
|
11
|
+
# VARIABLE were removed here -- grep-confirmed, before removal, to be
|
|
12
|
+
# produced by neither parser (codepulse/parser/python.py,
|
|
13
|
+
# codepulse/parser/javascript.py) and referenced nowhere else in the
|
|
14
|
+
# codebase or test suite, and confirmed absent from every row in the
|
|
15
|
+
# real codepulse.db's own symbols table. If a future parser needs to
|
|
16
|
+
# classify variables or interfaces as their own symbol kind, add the
|
|
17
|
+
# value back deliberately at that point, backed by a real producer,
|
|
18
|
+
# rather than carrying an always-empty classification indefinitely.
|
|
19
|
+
FUNCTION = "function"
|
|
20
|
+
METHOD = "method"
|
|
21
|
+
CLASS = "class"
|
|
22
|
+
MODULE = "module"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Symbol(BaseModel):
|
|
26
|
+
"""Represents an extracted code symbol (function, method, class, etc.)."""
|
|
27
|
+
name: str
|
|
28
|
+
type: SymbolType
|
|
29
|
+
file_path: str
|
|
30
|
+
start_line: int = Field(ge=1, description="1-indexed starting line")
|
|
31
|
+
end_line: int = Field(ge=1, description="1-indexed ending line")
|
|
32
|
+
signature: str = ""
|
|
33
|
+
docstring: Optional[str] = None
|
|
34
|
+
body: str = ""
|
|
35
|
+
body_hash: str = ""
|
|
36
|
+
language: str
|
|
37
|
+
parent_symbol: Optional[str] = None
|
|
38
|
+
complexity: int = 1
|
|
39
|
+
# Phase 4.1 (roadmap §37 item 4.1: "Base-class/inheritance extraction
|
|
40
|
+
# (Python + JS/TS)"; §29: "Symbol... needs... base-class field").
|
|
41
|
+
# Raw, unresolved base-class expressions exactly as written in source
|
|
42
|
+
# (e.g. ["Base", "pkg.Mixin"]) -- populated only for CLASS symbols,
|
|
43
|
+
# always empty for every other symbol type. This is the honest,
|
|
44
|
+
# never-fabricated extraction; RESOLVING each name to a real symbol
|
|
45
|
+
# (with confidence) is a separate, derived step performed by
|
|
46
|
+
# `CallGraphBuilder` -- the exact same layering already established
|
|
47
|
+
# for `CallReference.callee_name` (raw) vs. `CallGraphEdge` (resolved,
|
|
48
|
+
# confidence-scored). A `keyword_argument` in a Python class's base
|
|
49
|
+
# list (e.g. `metaclass=Meta`) is deliberately excluded here -- it is
|
|
50
|
+
# not a base class. A TypeScript `implements` clause is also
|
|
51
|
+
# deliberately excluded -- interface implementation is a different
|
|
52
|
+
# relationship than base-class inheritance, not extracted in this
|
|
53
|
+
# milestone.
|
|
54
|
+
base_classes: list[str] = Field(default_factory=list)
|
|
55
|
+
|
|
56
|
+
def model_post_init(self, __context) -> None:
|
|
57
|
+
if not self.body_hash and self.body:
|
|
58
|
+
self.body_hash = hashlib.sha256(self.body.strip().encode("utf-8")).hexdigest()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class ImportItem(BaseModel):
|
|
62
|
+
"""An imported item with an optional alias (e.g., `from math import sqrt as s`)."""
|
|
63
|
+
name: str
|
|
64
|
+
alias: Optional[str] = None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class ImportStatement(BaseModel):
|
|
68
|
+
"""Represents an import in source code."""
|
|
69
|
+
module: str
|
|
70
|
+
items: list[ImportItem] = Field(default_factory=list)
|
|
71
|
+
is_relative: bool = False
|
|
72
|
+
level: int = 0 # Number of leading dots in relative imports
|
|
73
|
+
line_number: int = 1
|
|
74
|
+
file_path: str = ""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class CallReference(BaseModel):
|
|
78
|
+
"""A reference where a symbol/function is invoked."""
|
|
79
|
+
caller_symbol: str
|
|
80
|
+
callee_name: str
|
|
81
|
+
line_number: int
|
|
82
|
+
file_path: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class CallGraphEdge(BaseModel):
|
|
86
|
+
"""A resolved or partially resolved call graph edge between two symbols."""
|
|
87
|
+
source_symbol_name: str
|
|
88
|
+
source_file: str
|
|
89
|
+
target_symbol_name: str
|
|
90
|
+
target_file: Optional[str] = None
|
|
91
|
+
call_count: int = 1
|
|
92
|
+
# Phase 3.4 (roadmap §37 item 3.4; §11): "high" -- resolved through an
|
|
93
|
+
# explicit import mapping to a known file+symbol; "low" -- resolved
|
|
94
|
+
# only by bare-name match against same-file symbols; "unresolved" --
|
|
95
|
+
# the callee couldn't be matched to any known symbol at all. Always
|
|
96
|
+
# set explicitly by CallGraphBuilder._build_graph (see its own
|
|
97
|
+
# comments) -- the "unresolved" default here is a safe fallback for
|
|
98
|
+
# any other construction path, never itself a claim of resolution.
|
|
99
|
+
# May be None ONLY for a legacy durable `call_edges` row written before
|
|
100
|
+
# Phase 3.4 added confidence (restored by `CallGraphBuilder.from_edges`);
|
|
101
|
+
# a graph built from parse results always sets it explicitly.
|
|
102
|
+
confidence: Optional[str] = "unresolved"
|
|
103
|
+
# Phase 4.1 (roadmap §37 item 4.1): "calls" (default, matching every
|
|
104
|
+
# edge this project has ever persisted before this field existed) or
|
|
105
|
+
# "inherits" (a resolved base-class relationship). Always set
|
|
106
|
+
# explicitly by CallGraphBuilder._build_graph for both kinds -- the
|
|
107
|
+
# "calls" default here is a safe fallback for any other construction
|
|
108
|
+
# path, matching this project's existing historical data exactly.
|
|
109
|
+
relationship_type: str = "calls"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class FileParseResult(BaseModel):
|
|
113
|
+
"""Complete AST parse result for a single file."""
|
|
114
|
+
file_path: str
|
|
115
|
+
language: str
|
|
116
|
+
symbols: list[Symbol] = Field(default_factory=list)
|
|
117
|
+
imports: list[ImportStatement] = Field(default_factory=list)
|
|
118
|
+
calls: list[CallReference] = Field(default_factory=list)
|
|
119
|
+
file_size_bytes: int = 0
|
|
120
|
+
line_count: int = 0
|
|
121
|
+
content_hash: str = Field(
|
|
122
|
+
default="",
|
|
123
|
+
description=(
|
|
124
|
+
"SHA-256 hex digest of the file's raw source content, used as the "
|
|
125
|
+
"change-detection key for idempotent File upsert (Phase 0.3)."
|
|
126
|
+
),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def compute_content_hash(content: str) -> str:
|
|
131
|
+
"""
|
|
132
|
+
Computes a deterministic SHA-256 hex digest for whole-file content.
|
|
133
|
+
|
|
134
|
+
Follows the same hashing convention already used for `Symbol.body_hash`
|
|
135
|
+
(SHA-256 over UTF-8-encoded text). Unlike `Symbol.body_hash`, the content
|
|
136
|
+
is NOT stripped first: a file's identity hash must reflect the exact
|
|
137
|
+
persisted bytes so that whitespace-only edits still register as a real
|
|
138
|
+
content change during incremental re-indexing.
|
|
139
|
+
"""
|
|
140
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Phase 7.4 -- Observability wiring (roadmap §37 item 7.4; §18
|
|
3
|
+
Observability Architecture: "actually wire the already-declared
|
|
4
|
+
dependency" (`prometheus-client`, `structlog`) -- both were declared in
|
|
5
|
+
`pyproject.toml` since Phase 0.9 but never installed or imported by any
|
|
6
|
+
code until this milestone; both are now genuinely installed and used).
|
|
7
|
+
|
|
8
|
+
§18, quoted and implemented exactly:
|
|
9
|
+
- **Metrics** (real Prometheus counters/histograms/gauge, named
|
|
10
|
+
exactly as listed): `search_requests_total{search_type}`,
|
|
11
|
+
`search_latency_seconds`, `indexing_jobs_total{status}`,
|
|
12
|
+
`indexing_duration_seconds`, `files_indexed_total`,
|
|
13
|
+
`symbols_indexed_total`, `embedding_duration_seconds`,
|
|
14
|
+
`job_queue_depth`, `api_errors_total{endpoint,status_code}`,
|
|
15
|
+
`search_zero_result_total`.
|
|
16
|
+
- **Logs**: structured JSON via `structlog`, with the exact field
|
|
17
|
+
set §18 names: `request_id`, `repository_id`, `job_id`,
|
|
18
|
+
`api_key_id`, `operation`, `duration_ms`, `error_type`.
|
|
19
|
+
- **Health endpoints** (`codepulse/api/app.py`): `/health/live`
|
|
20
|
+
(process up) and `/health/ready` (DB reachable, embedding model
|
|
21
|
+
loaded, index storage writable) -- built in `app.py` itself using
|
|
22
|
+
this module's `check_readiness`, kept out of this module because
|
|
23
|
+
they need live references to `storage_manager`/`indexing_engine`.
|
|
24
|
+
|
|
25
|
+
This module holds a DEDICATED `CollectorRegistry` (not
|
|
26
|
+
`prometheus_client`'s global default) so importing it repeatedly across
|
|
27
|
+
this project's own test suite (many isolated test modules, potentially
|
|
28
|
+
several `TestClient(app)` instances per test session) can never raise
|
|
29
|
+
"Duplicated timeseries in CollectorRegistry" -- module-level singletons
|
|
30
|
+
are created exactly once per Python process regardless of how many
|
|
31
|
+
tests import this module, which is what makes a dedicated registry
|
|
32
|
+
unnecessary in principle, but using one anyway is the standard,
|
|
33
|
+
recommended `prometheus_client` pattern for avoiding any accidental
|
|
34
|
+
collision with a DIFFERENT library's own use of the global default
|
|
35
|
+
registry in the same process.
|
|
36
|
+
|
|
37
|
+
Every counter/histogram below is incremented/observed from a REAL,
|
|
38
|
+
already-happening event (a request actually being served, a job
|
|
39
|
+
actually transitioning state, an embedding call actually completing) --
|
|
40
|
+
never a fabricated/synthetic value, per CLAUDE.md's "no fabricated
|
|
41
|
+
metrics" invariant applied to observability data too.
|
|
42
|
+
"""
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import logging
|
|
46
|
+
import sys
|
|
47
|
+
from typing import Any
|
|
48
|
+
|
|
49
|
+
import structlog
|
|
50
|
+
from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, Gauge, Histogram, generate_latest
|
|
51
|
+
|
|
52
|
+
REGISTRY = CollectorRegistry()
|
|
53
|
+
|
|
54
|
+
search_requests_total = Counter(
|
|
55
|
+
"search_requests_total", "Total search requests served, by search_type", ["search_type"], registry=REGISTRY
|
|
56
|
+
)
|
|
57
|
+
search_latency_seconds = Histogram(
|
|
58
|
+
"search_latency_seconds", "Search request latency in seconds", registry=REGISTRY
|
|
59
|
+
)
|
|
60
|
+
search_zero_result_total = Counter(
|
|
61
|
+
"search_zero_result_total", "Total search requests that returned zero results", registry=REGISTRY
|
|
62
|
+
)
|
|
63
|
+
indexing_jobs_total = Counter(
|
|
64
|
+
"indexing_jobs_total", "Total indexing jobs, by terminal/transition status", ["status"], registry=REGISTRY
|
|
65
|
+
)
|
|
66
|
+
indexing_duration_seconds = Histogram(
|
|
67
|
+
"indexing_duration_seconds", "Wall-clock duration of one index_directory() run, in seconds", registry=REGISTRY
|
|
68
|
+
)
|
|
69
|
+
files_indexed_total = Counter(
|
|
70
|
+
"files_indexed_total", "Total files processed across all indexing runs", registry=REGISTRY
|
|
71
|
+
)
|
|
72
|
+
symbols_indexed_total = Counter(
|
|
73
|
+
"symbols_indexed_total", "Total symbols processed across all indexing runs", registry=REGISTRY
|
|
74
|
+
)
|
|
75
|
+
embedding_duration_seconds = Histogram(
|
|
76
|
+
"embedding_duration_seconds", "Duration of one CodeEmbedder.encode() call, in seconds", registry=REGISTRY
|
|
77
|
+
)
|
|
78
|
+
job_queue_depth = Gauge(
|
|
79
|
+
"job_queue_depth", "Current number of queued+running IndexJob rows", registry=REGISTRY
|
|
80
|
+
)
|
|
81
|
+
api_errors_total = Counter(
|
|
82
|
+
"api_errors_total", "Total error responses, by endpoint path and status code", ["endpoint", "status_code"], registry=REGISTRY
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def render_metrics() -> tuple[bytes, str]:
|
|
87
|
+
"""Returns `(body, content_type)` for the `/metrics` endpoint -- the real, current state of `REGISTRY`, generated fresh on every call (never cached/stale)."""
|
|
88
|
+
return generate_latest(REGISTRY), CONTENT_TYPE_LATEST
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
_logging_configured = False
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def configure_logging() -> None:
|
|
95
|
+
"""
|
|
96
|
+
Configures `structlog` for JSON-rendered structured log output, per
|
|
97
|
+
§18: "structured (JSON via structlog)... every log line carries
|
|
98
|
+
enough to reconstruct a request's full path." Idempotent -- safe to
|
|
99
|
+
call more than once (e.g. once per test module importing
|
|
100
|
+
`codepulse.api.app`), only actually reconfigures the first time.
|
|
101
|
+
"""
|
|
102
|
+
global _logging_configured
|
|
103
|
+
if _logging_configured:
|
|
104
|
+
return
|
|
105
|
+
structlog.configure(
|
|
106
|
+
processors=[
|
|
107
|
+
structlog.processors.TimeStamper(fmt="iso"),
|
|
108
|
+
structlog.processors.add_log_level,
|
|
109
|
+
structlog.processors.StackInfoRenderer(),
|
|
110
|
+
structlog.processors.format_exc_info,
|
|
111
|
+
structlog.processors.JSONRenderer(),
|
|
112
|
+
],
|
|
113
|
+
logger_factory=structlog.PrintLoggerFactory(file=sys.stdout),
|
|
114
|
+
wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
|
|
115
|
+
cache_logger_on_first_use=True,
|
|
116
|
+
)
|
|
117
|
+
_logging_configured = True
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def get_logger() -> Any:
|
|
121
|
+
"""
|
|
122
|
+
Returns a structlog bound logger. §18's own exact field set
|
|
123
|
+
(`request_id`, `repository_id`, `job_id`, `api_key_id`,
|
|
124
|
+
`operation`, `duration_ms`, `error_type`) is bound by each CALL
|
|
125
|
+
SITE via `.bind(...)`/keyword arguments -- this function does not
|
|
126
|
+
hardcode which fields a given log line carries, since not every
|
|
127
|
+
operation has all of them (e.g. a search request has no `job_id`).
|
|
128
|
+
"""
|
|
129
|
+
configure_logging()
|
|
130
|
+
return structlog.get_logger()
|
codepulse/parser/base.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from codepulse.models import (
|
|
5
|
+
FileParseResult,
|
|
6
|
+
ImportStatement,
|
|
7
|
+
Symbol,
|
|
8
|
+
CallReference,
|
|
9
|
+
compute_content_hash,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class LanguageParser(ABC):
|
|
14
|
+
"""Abstract Base Class for language-specific AST parsers."""
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def language(self) -> str:
|
|
19
|
+
"""Returns the canonical language identifier (e.g. 'python', 'javascript')."""
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def extract_symbols(self, source_code: str, file_path: str = "") -> list[Symbol]:
|
|
24
|
+
"""Extract functions, classes, methods, and types with their signatures and docstrings."""
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def extract_imports(self, source_code: str, file_path: str = "") -> list[ImportStatement]:
|
|
29
|
+
"""Extract module imports and imported identifiers."""
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
@abstractmethod
|
|
33
|
+
def extract_calls(
|
|
34
|
+
self, source_code: str, file_path: str = "", symbols: list[Symbol] | None = None
|
|
35
|
+
) -> list[CallReference]:
|
|
36
|
+
"""Extract function/method invocations mapped to their enclosing caller symbols."""
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
def parse_file(self, file_path: str, source_code: str) -> FileParseResult:
|
|
40
|
+
"""Parse source code completely and return aggregated FileParseResult."""
|
|
41
|
+
symbols = self.extract_symbols(source_code, file_path=file_path)
|
|
42
|
+
imports = self.extract_imports(source_code, file_path=file_path)
|
|
43
|
+
calls = self.extract_calls(source_code, file_path=file_path, symbols=symbols)
|
|
44
|
+
|
|
45
|
+
line_count = len(source_code.splitlines())
|
|
46
|
+
file_size_bytes = len(source_code.encode("utf-8"))
|
|
47
|
+
content_hash = compute_content_hash(source_code)
|
|
48
|
+
|
|
49
|
+
return FileParseResult(
|
|
50
|
+
file_path=file_path,
|
|
51
|
+
language=self.language,
|
|
52
|
+
symbols=symbols,
|
|
53
|
+
imports=imports,
|
|
54
|
+
calls=calls,
|
|
55
|
+
file_size_bytes=file_size_bytes,
|
|
56
|
+
line_count=line_count,
|
|
57
|
+
content_hash=content_hash,
|
|
58
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from codepulse.parser.base import LanguageParser
|
|
5
|
+
from codepulse.parser.python import PythonParser
|
|
6
|
+
from codepulse.parser.javascript import JavaScriptParser
|
|
7
|
+
|
|
8
|
+
_PARSERS: dict[str, LanguageParser] = {}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_parser_for_language(language: str) -> LanguageParser:
|
|
12
|
+
"""Retrieve a cached parser instance for the requested language."""
|
|
13
|
+
lang_lower = language.lower().strip()
|
|
14
|
+
if lang_lower not in _PARSERS:
|
|
15
|
+
if lang_lower in ("python", "py"):
|
|
16
|
+
_PARSERS[lang_lower] = PythonParser()
|
|
17
|
+
elif lang_lower in ("javascript", "js"):
|
|
18
|
+
_PARSERS[lang_lower] = JavaScriptParser(is_typescript=False)
|
|
19
|
+
elif lang_lower in ("typescript", "ts"):
|
|
20
|
+
_PARSERS[lang_lower] = JavaScriptParser(is_typescript=True)
|
|
21
|
+
else:
|
|
22
|
+
raise ValueError(f"Unsupported language: '{language}'")
|
|
23
|
+
return _PARSERS[lang_lower]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_parser_for_file(file_path: str) -> LanguageParser | None:
|
|
27
|
+
"""Determine the appropriate parser based on file extension."""
|
|
28
|
+
_, ext = os.path.splitext(file_path)
|
|
29
|
+
ext_lower = ext.lower()
|
|
30
|
+
|
|
31
|
+
if ext_lower == ".py":
|
|
32
|
+
return get_parser_for_language("python")
|
|
33
|
+
elif ext_lower in (".js", ".jsx", ".mjs", ".cjs"):
|
|
34
|
+
return get_parser_for_language("javascript")
|
|
35
|
+
elif ext_lower in (".ts", ".tsx", ".mts", ".cts"):
|
|
36
|
+
return get_parser_for_language("typescript")
|
|
37
|
+
return None
|