codepulse-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. codepulse/__init__.py +5 -0
  2. codepulse/api/app.py +1336 -0
  3. codepulse/api/errors.py +172 -0
  4. codepulse/api/security.py +189 -0
  5. codepulse/auth.py +406 -0
  6. codepulse/cli.py +1052 -0
  7. codepulse/config.py +475 -0
  8. codepulse/data_location.py +78 -0
  9. codepulse/embeddings/chunker.py +138 -0
  10. codepulse/embeddings/embedder.py +120 -0
  11. codepulse/git_diff.py +201 -0
  12. codepulse/indexer/engine.py +2655 -0
  13. codepulse/indexer/ignore.py +120 -0
  14. codepulse/ingestion_safety.py +169 -0
  15. codepulse/jobs.py +257 -0
  16. codepulse/mcp_server.py +269 -0
  17. codepulse/models.py +140 -0
  18. codepulse/observability.py +130 -0
  19. codepulse/parser/__init__.py +8 -0
  20. codepulse/parser/base.py +58 -0
  21. codepulse/parser/factory.py +37 -0
  22. codepulse/parser/graph.py +502 -0
  23. codepulse/parser/javascript.py +428 -0
  24. codepulse/parser/module_graph.py +232 -0
  25. codepulse/parser/python.py +331 -0
  26. codepulse/rate_limit.py +142 -0
  27. codepulse/repository_identity.py +83 -0
  28. codepulse/search/bm25.py +342 -0
  29. codepulse/search/ranker.py +382 -0
  30. codepulse/search/tokenizer.py +80 -0
  31. codepulse/search/vector.py +357 -0
  32. codepulse/storage/db.py +526 -0
  33. codepulse/storage/migrations.py +347 -0
  34. codepulse/storage/schema_migrations/README +1 -0
  35. codepulse/storage/schema_migrations/env.py +70 -0
  36. codepulse/storage/schema_migrations/script.py.mako +28 -0
  37. codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
  38. codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
  39. codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
  40. codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
  41. codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
  42. codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
  43. codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
  44. codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
  45. codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
  46. codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
  47. codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
  48. codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
  49. codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
  50. codepulse/symbol_identity.py +29 -0
  51. codepulse_engine-0.1.0.dist-info/METADATA +191 -0
  52. codepulse_engine-0.1.0.dist-info/RECORD +56 -0
  53. codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
  54. codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
  55. codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
  56. codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,269 @@
1
+ """
2
+ Phase 5.3 -- MCP server (roadmap §37 item 5.3; §13 MCP Strategy).
3
+
4
+ §13, quoted and held to strictly: *"Recommendation: yes, expose an MCP
5
+ server -- but strictly as an adapter, not a parallel implementation.
6
+ Each MCP tool calls the exact same internal Python functions the REST
7
+ API calls."* *"Proposed tools: search_code, get_symbol, find_callers,
8
+ find_callees, impact_analysis, find_similar, get_context (the §12
9
+ bundle)."* *"Permissions/repository selection: MCP connections are
10
+ scoped to one or more repositories at connection-setup time (via the
11
+ same API-key/permission model as REST -- §16), not per-tool-call."*
12
+ *"Failure behavior: tool calls return structured errors
13
+ ({"error": "repository_not_indexed", "message": "..."}) rather than
14
+ throwing opaque exceptions."*
15
+
16
+ Architecture: this module is a THIN adapter over `IndexingEngine` and
17
+ `codepulse.auth`/`codepulse.api.security`'s existing authorization
18
+ primitives -- it introduces no new graph/retrieval/authorization logic
19
+ of its own. `build_mcp_server(storage, engine, api_key)` resolves the
20
+ given API key to an `AuthorizedKey` exactly ONCE, via the same
21
+ `verify_api_key` (Phase 1.1) function REST's own `require_api_key`
22
+ dependency calls -- not a second authentication mechanism -- and every
23
+ tool call authorizes against that SAME resolved `repository_ids` set,
24
+ never re-authenticating per call (per §13's own explicit "not
25
+ per-tool-call" instruction; FastAPI's per-request `Depends(
26
+ require_api_key)` mechanism does not apply here since MCP tool calls
27
+ are not individual HTTP requests).
28
+
29
+ Each tool below calls the exact same `IndexingEngine` method its REST
30
+ counterpart calls (`search`, `analyze_impact`, `find_similar_symbols`,
31
+ `get_context_bundle`) -- `find_callers`/`find_callees` both reuse
32
+ `analyze_impact` internally (there is no dedicated standalone
33
+ REST/engine method for either; §13 names them as separate TOOLS, not
34
+ as evidence of a separate implementation, so they are built as thin
35
+ projections of the one already-tested `analyze_impact` result rather
36
+ than a second call-graph traversal). `get_symbol`'s `called_by`/`calls`
37
+ query mirrors `GET /v1/symbols/{id}`'s own tested query exactly
38
+ (including the Phase 4.1 compatibility correction's `relationship_type
39
+ == "calls"` filter) -- a small, deliberate, disclosed duplication of a
40
+ plain parameterized DB read (not a graph algorithm), not extracted into
41
+ a third shared location this milestone; see CODEPULSE_PROGRESS.md's
42
+ Phase 5.3 entry for the explicit reasoning.
43
+
44
+ Backend-hardening addition (not a roadmap sprint item; §13's own
45
+ literal tool list names exactly 7): `module_structure` -- an 8th tool,
46
+ giving Phase 4.3's `IndexingEngine.analyze_module_structure` (built
47
+ with no MCP caller at the time, by that milestone's own explicit
48
+ approved scope) its first MCP surface, matching the REST/CLI query
49
+ surfaces added in the same pass.
50
+ """
51
+ from typing import Any, Optional
52
+
53
+ from mcp.server.fastmcp import FastMCP
54
+
55
+ from codepulse.auth import verify_api_key
56
+ from codepulse.api.security import AuthorizedKey
57
+ from codepulse.indexer.engine import IndexingEngine
58
+ from codepulse.storage.db import APIKeyRepositoryModel, CallEdge, StorageManager, SymbolModel
59
+
60
+
61
+ class MCPAuthenticationError(Exception):
62
+ """Raised by `build_mcp_server` when the supplied API key is missing, unknown, or revoked."""
63
+
64
+
65
+ def _structured_error(code: str, message: str) -> dict[str, str]:
66
+ """§13's own literal failure-behavior contract: `{"error": "...", "message": "..."}`."""
67
+ return {"error": code, "message": message}
68
+
69
+
70
+ def build_mcp_server(
71
+ storage: StorageManager, engine: IndexingEngine, api_key: str, name: str = "codepulse"
72
+ ) -> FastMCP:
73
+ """
74
+ Builds and returns a `FastMCP` server instance scoped to the
75
+ repositories the given `api_key` is authorized for -- resolved
76
+ ONCE, at connection-setup time, per §13's explicit instruction.
77
+ Raises `MCPAuthenticationError` for a missing/unknown/revoked key,
78
+ mirroring `require_api_key`'s own HTTP 401 case (there is no HTTP
79
+ response here to return, so this is a plain exception the caller
80
+ -- e.g. `codepulse serve`, Phase 6 -- decides how to report).
81
+ """
82
+ api_key_id = verify_api_key(storage, api_key)
83
+ if api_key_id is None:
84
+ raise MCPAuthenticationError("Invalid or revoked API key")
85
+
86
+ session = storage.get_session()
87
+ try:
88
+ repo_ids = {
89
+ row.repo_id
90
+ for row in session.query(APIKeyRepositoryModel)
91
+ .filter(APIKeyRepositoryModel.api_key_id == api_key_id)
92
+ .all()
93
+ }
94
+ finally:
95
+ session.close()
96
+
97
+ authorized = AuthorizedKey(api_key_id=api_key_id, repository_ids=frozenset(repo_ids))
98
+
99
+ mcp: FastMCP = FastMCP(name)
100
+
101
+ def _authorize(repo_id: int) -> Optional[dict[str, str]]:
102
+ if repo_id not in authorized.repository_ids:
103
+ return _structured_error(
104
+ "repository_not_authorized",
105
+ f"This connection is not authorized for repository {repo_id}.",
106
+ )
107
+ return None
108
+
109
+ def _load_symbol(symbol_id: int) -> Optional[dict[str, Any]]:
110
+ """Extracts every field a tool might need into a plain dict
111
+ WHILE the session is open (SQLAlchemy relationship access after
112
+ `session.close()` would raise `DetachedInstanceError`), exactly
113
+ matching the "access `sym.file` before closing" pattern every
114
+ REST symbol route already follows."""
115
+ db_session = storage.get_session()
116
+ try:
117
+ sym = db_session.query(SymbolModel).filter(SymbolModel.id == symbol_id).first()
118
+ if not sym or not sym.file:
119
+ return None
120
+ return {
121
+ "id": sym.id,
122
+ "name": sym.name,
123
+ "type": sym.type,
124
+ "file_path": sym.file.path,
125
+ "repo_id": sym.file.repo_id,
126
+ "signature": sym.signature,
127
+ "docstring": sym.docstring,
128
+ "body": sym.body,
129
+ "complexity": sym.complexity,
130
+ }
131
+ finally:
132
+ db_session.close()
133
+
134
+ @mcp.tool()
135
+ def search_code(
136
+ query: str, repository_id: int, limit: int = 20, search_type: str = "hybrid"
137
+ ) -> dict[str, Any]:
138
+ """Hybrid/keyword/semantic/RRF search over an indexed repository's code, scoped to this connection's authorized repositories."""
139
+ auth_error = _authorize(repository_id)
140
+ if auth_error:
141
+ return auth_error
142
+ results = engine.search(query, limit=limit, search_type=search_type, repository_id=repository_id)
143
+ return {"results": [r.to_dict() for r in results]}
144
+
145
+ @mcp.tool()
146
+ def get_symbol(symbol_id: int) -> dict[str, Any]:
147
+ """Fetches one symbol's definition plus its direct callers/callees (calls-only, matching GET /v1/symbols/{id})."""
148
+ loaded = _load_symbol(symbol_id)
149
+ if loaded is None:
150
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
151
+ auth_error = _authorize(loaded["repo_id"])
152
+ if auth_error:
153
+ return auth_error
154
+
155
+ db_session = storage.get_session()
156
+ try:
157
+ called_by = (
158
+ db_session.query(CallEdge)
159
+ .filter(CallEdge.target_symbol_name == loaded["name"])
160
+ .filter(CallEdge.repo_id == loaded["repo_id"])
161
+ .filter(CallEdge.relationship_type == "calls")
162
+ .all()
163
+ )
164
+ calls = (
165
+ db_session.query(CallEdge)
166
+ .filter(CallEdge.source_symbol_name == loaded["name"])
167
+ .filter(CallEdge.repo_id == loaded["repo_id"])
168
+ .filter(CallEdge.relationship_type == "calls")
169
+ .all()
170
+ )
171
+ finally:
172
+ db_session.close()
173
+
174
+ return {
175
+ "symbol_id": loaded["id"],
176
+ "name": loaded["name"],
177
+ "type": loaded["type"],
178
+ "file": loaded["file_path"],
179
+ "docstring": loaded["docstring"],
180
+ "source_code": loaded["body"],
181
+ "called_by": [
182
+ {"symbol": d.source_symbol_name, "file": d.source_file, "confidence": d.confidence}
183
+ for d in called_by
184
+ ],
185
+ "calls": [
186
+ {"symbol": d.target_symbol_name, "file": d.target_file, "confidence": d.confidence}
187
+ for d in calls
188
+ ],
189
+ "complexity": loaded["complexity"],
190
+ }
191
+
192
+ @mcp.tool()
193
+ def find_callers(symbol_id: int, depth: int = 3) -> dict[str, Any]:
194
+ """Symbols that call this one, direct and transitive, confidence-scored and depth-bucketed (a projection of impact_analysis's own callers side)."""
195
+ loaded = _load_symbol(symbol_id)
196
+ if loaded is None:
197
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
198
+ auth_error = _authorize(loaded["repo_id"])
199
+ if auth_error:
200
+ return auth_error
201
+ impact = engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
202
+ return {"symbol": loaded["name"], "callers": impact["call_chain"]}
203
+
204
+ @mcp.tool()
205
+ def find_callees(symbol_id: int, depth: int = 3) -> dict[str, Any]:
206
+ """Symbols this one calls, direct and transitive, confidence-scored and depth-bucketed (a projection of impact_analysis's own callees side)."""
207
+ loaded = _load_symbol(symbol_id)
208
+ if loaded is None:
209
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
210
+ auth_error = _authorize(loaded["repo_id"])
211
+ if auth_error:
212
+ return auth_error
213
+ impact = engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
214
+ return {"symbol": loaded["name"], "callees": impact["dependencies"]}
215
+
216
+ @mcp.tool()
217
+ def impact_analysis(symbol_id: int, depth: int = 3) -> dict[str, Any]:
218
+ """Direct-vs-transitive impact bucketing for a symbol (calls only; see CODEPULSE_PROGRESS.md's Phase 4.2 Decision 1)."""
219
+ loaded = _load_symbol(symbol_id)
220
+ if loaded is None:
221
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
222
+ auth_error = _authorize(loaded["repo_id"])
223
+ if auth_error:
224
+ return auth_error
225
+ return engine.analyze_impact(loaded["name"], loaded["file_path"], loaded["repo_id"], max_depth=depth)
226
+
227
+ @mcp.tool()
228
+ def find_similar(symbol_id: int, threshold: float = 0.75, limit: int = 10) -> dict[str, Any]:
229
+ """Semantically similar symbols within the SAME repository (embedding-based, repository-scoped)."""
230
+ loaded = _load_symbol(symbol_id)
231
+ if loaded is None:
232
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
233
+ auth_error = _authorize(loaded["repo_id"])
234
+ if auth_error:
235
+ return auth_error
236
+ query_text = f"{loaded['signature']}\n{loaded['docstring'] or ''}\n{loaded['body'][:500]}"
237
+ similar = engine.find_similar_symbols(query_text, loaded["name"], loaded["repo_id"], threshold=threshold, limit=limit)
238
+ return {"query_symbol": {"id": loaded["id"], "name": loaded["name"]}, "similar": similar}
239
+
240
+ @mcp.tool()
241
+ def get_context(symbol_id: int, include_body: bool = False, max_related: int = 10) -> dict[str, Any]:
242
+ """The §12 agent-native context bundle: definition, direct callers/callees, semantically related symbols, snapshot commit, and a truncation flag."""
243
+ loaded = _load_symbol(symbol_id)
244
+ if loaded is None:
245
+ return _structured_error("symbol_not_found", f"No symbol with id {symbol_id}.")
246
+ auth_error = _authorize(loaded["repo_id"])
247
+ if auth_error:
248
+ return auth_error
249
+ return engine.get_context_bundle(
250
+ loaded["name"], loaded["file_path"], loaded["repo_id"],
251
+ signature=loaded["signature"], docstring=loaded["docstring"], body=loaded["body"],
252
+ include_body=include_body, max_related=max_related,
253
+ )
254
+
255
+ @mcp.tool()
256
+ def module_structure(repository_id: int) -> dict[str, Any]:
257
+ """Module clusters + raw inbound/outbound degree rankings for a repository's file-level import graph (Phase 4.3's ModuleGraphBuilder, reused unmodified)."""
258
+ auth_error = _authorize(repository_id)
259
+ if auth_error:
260
+ return auth_error
261
+ module_graph = engine.analyze_module_structure(repository_id)
262
+ return {
263
+ "repository_id": repository_id,
264
+ "clusters": module_graph.clusters,
265
+ "ranked_by_inbound": module_graph.rank_by_inbound(),
266
+ "ranked_by_outbound": module_graph.rank_by_outbound(),
267
+ }
268
+
269
+ return mcp
codepulse/models.py ADDED
@@ -0,0 +1,140 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ from enum import Enum
5
+ from typing import Optional
6
+ from pydantic import BaseModel, Field
7
+
8
+
9
+ class SymbolType(str, Enum):
10
+ # Backend-hardening cleanup (not a roadmap sprint item): INTERFACE and
11
+ # VARIABLE were removed here -- grep-confirmed, before removal, to be
12
+ # produced by neither parser (codepulse/parser/python.py,
13
+ # codepulse/parser/javascript.py) and referenced nowhere else in the
14
+ # codebase or test suite, and confirmed absent from every row in the
15
+ # real codepulse.db's own symbols table. If a future parser needs to
16
+ # classify variables or interfaces as their own symbol kind, add the
17
+ # value back deliberately at that point, backed by a real producer,
18
+ # rather than carrying an always-empty classification indefinitely.
19
+ FUNCTION = "function"
20
+ METHOD = "method"
21
+ CLASS = "class"
22
+ MODULE = "module"
23
+
24
+
25
+ class Symbol(BaseModel):
26
+ """Represents an extracted code symbol (function, method, class, etc.)."""
27
+ name: str
28
+ type: SymbolType
29
+ file_path: str
30
+ start_line: int = Field(ge=1, description="1-indexed starting line")
31
+ end_line: int = Field(ge=1, description="1-indexed ending line")
32
+ signature: str = ""
33
+ docstring: Optional[str] = None
34
+ body: str = ""
35
+ body_hash: str = ""
36
+ language: str
37
+ parent_symbol: Optional[str] = None
38
+ complexity: int = 1
39
+ # Phase 4.1 (roadmap §37 item 4.1: "Base-class/inheritance extraction
40
+ # (Python + JS/TS)"; §29: "Symbol... needs... base-class field").
41
+ # Raw, unresolved base-class expressions exactly as written in source
42
+ # (e.g. ["Base", "pkg.Mixin"]) -- populated only for CLASS symbols,
43
+ # always empty for every other symbol type. This is the honest,
44
+ # never-fabricated extraction; RESOLVING each name to a real symbol
45
+ # (with confidence) is a separate, derived step performed by
46
+ # `CallGraphBuilder` -- the exact same layering already established
47
+ # for `CallReference.callee_name` (raw) vs. `CallGraphEdge` (resolved,
48
+ # confidence-scored). A `keyword_argument` in a Python class's base
49
+ # list (e.g. `metaclass=Meta`) is deliberately excluded here -- it is
50
+ # not a base class. A TypeScript `implements` clause is also
51
+ # deliberately excluded -- interface implementation is a different
52
+ # relationship than base-class inheritance, not extracted in this
53
+ # milestone.
54
+ base_classes: list[str] = Field(default_factory=list)
55
+
56
+ def model_post_init(self, __context) -> None:
57
+ if not self.body_hash and self.body:
58
+ self.body_hash = hashlib.sha256(self.body.strip().encode("utf-8")).hexdigest()
59
+
60
+
61
+ class ImportItem(BaseModel):
62
+ """An imported item with an optional alias (e.g., `from math import sqrt as s`)."""
63
+ name: str
64
+ alias: Optional[str] = None
65
+
66
+
67
+ class ImportStatement(BaseModel):
68
+ """Represents an import in source code."""
69
+ module: str
70
+ items: list[ImportItem] = Field(default_factory=list)
71
+ is_relative: bool = False
72
+ level: int = 0 # Number of leading dots in relative imports
73
+ line_number: int = 1
74
+ file_path: str = ""
75
+
76
+
77
+ class CallReference(BaseModel):
78
+ """A reference where a symbol/function is invoked."""
79
+ caller_symbol: str
80
+ callee_name: str
81
+ line_number: int
82
+ file_path: str
83
+
84
+
85
+ class CallGraphEdge(BaseModel):
86
+ """A resolved or partially resolved call graph edge between two symbols."""
87
+ source_symbol_name: str
88
+ source_file: str
89
+ target_symbol_name: str
90
+ target_file: Optional[str] = None
91
+ call_count: int = 1
92
+ # Phase 3.4 (roadmap §37 item 3.4; §11): "high" -- resolved through an
93
+ # explicit import mapping to a known file+symbol; "low" -- resolved
94
+ # only by bare-name match against same-file symbols; "unresolved" --
95
+ # the callee couldn't be matched to any known symbol at all. Always
96
+ # set explicitly by CallGraphBuilder._build_graph (see its own
97
+ # comments) -- the "unresolved" default here is a safe fallback for
98
+ # any other construction path, never itself a claim of resolution.
99
+ # May be None ONLY for a legacy durable `call_edges` row written before
100
+ # Phase 3.4 added confidence (restored by `CallGraphBuilder.from_edges`);
101
+ # a graph built from parse results always sets it explicitly.
102
+ confidence: Optional[str] = "unresolved"
103
+ # Phase 4.1 (roadmap §37 item 4.1): "calls" (default, matching every
104
+ # edge this project has ever persisted before this field existed) or
105
+ # "inherits" (a resolved base-class relationship). Always set
106
+ # explicitly by CallGraphBuilder._build_graph for both kinds -- the
107
+ # "calls" default here is a safe fallback for any other construction
108
+ # path, matching this project's existing historical data exactly.
109
+ relationship_type: str = "calls"
110
+
111
+
112
+ class FileParseResult(BaseModel):
113
+ """Complete AST parse result for a single file."""
114
+ file_path: str
115
+ language: str
116
+ symbols: list[Symbol] = Field(default_factory=list)
117
+ imports: list[ImportStatement] = Field(default_factory=list)
118
+ calls: list[CallReference] = Field(default_factory=list)
119
+ file_size_bytes: int = 0
120
+ line_count: int = 0
121
+ content_hash: str = Field(
122
+ default="",
123
+ description=(
124
+ "SHA-256 hex digest of the file's raw source content, used as the "
125
+ "change-detection key for idempotent File upsert (Phase 0.3)."
126
+ ),
127
+ )
128
+
129
+
130
+ def compute_content_hash(content: str) -> str:
131
+ """
132
+ Computes a deterministic SHA-256 hex digest for whole-file content.
133
+
134
+ Follows the same hashing convention already used for `Symbol.body_hash`
135
+ (SHA-256 over UTF-8-encoded text). Unlike `Symbol.body_hash`, the content
136
+ is NOT stripped first: a file's identity hash must reflect the exact
137
+ persisted bytes so that whitespace-only edits still register as a real
138
+ content change during incremental re-indexing.
139
+ """
140
+ return hashlib.sha256(content.encode("utf-8")).hexdigest()
@@ -0,0 +1,130 @@
1
+ """
2
+ Phase 7.4 -- Observability wiring (roadmap §37 item 7.4; §18
3
+ Observability Architecture: "actually wire the already-declared
4
+ dependency" (`prometheus-client`, `structlog`) -- both were declared in
5
+ `pyproject.toml` since Phase 0.9 but never installed or imported by any
6
+ code until this milestone; both are now genuinely installed and used).
7
+
8
+ §18, quoted and implemented exactly:
9
+ - **Metrics** (real Prometheus counters/histograms/gauge, named
10
+ exactly as listed): `search_requests_total{search_type}`,
11
+ `search_latency_seconds`, `indexing_jobs_total{status}`,
12
+ `indexing_duration_seconds`, `files_indexed_total`,
13
+ `symbols_indexed_total`, `embedding_duration_seconds`,
14
+ `job_queue_depth`, `api_errors_total{endpoint,status_code}`,
15
+ `search_zero_result_total`.
16
+ - **Logs**: structured JSON via `structlog`, with the exact field
17
+ set §18 names: `request_id`, `repository_id`, `job_id`,
18
+ `api_key_id`, `operation`, `duration_ms`, `error_type`.
19
+ - **Health endpoints** (`codepulse/api/app.py`): `/health/live`
20
+ (process up) and `/health/ready` (DB reachable, embedding model
21
+ loaded, index storage writable) -- built in `app.py` itself using
22
+ this module's `check_readiness`, kept out of this module because
23
+ they need live references to `storage_manager`/`indexing_engine`.
24
+
25
+ This module holds a DEDICATED `CollectorRegistry` (not
26
+ `prometheus_client`'s global default) so importing it repeatedly across
27
+ this project's own test suite (many isolated test modules, potentially
28
+ several `TestClient(app)` instances per test session) can never raise
29
+ "Duplicated timeseries in CollectorRegistry" -- module-level singletons
30
+ are created exactly once per Python process regardless of how many
31
+ tests import this module, which is what makes a dedicated registry
32
+ unnecessary in principle, but using one anyway is the standard,
33
+ recommended `prometheus_client` pattern for avoiding any accidental
34
+ collision with a DIFFERENT library's own use of the global default
35
+ registry in the same process.
36
+
37
+ Every counter/histogram below is incremented/observed from a REAL,
38
+ already-happening event (a request actually being served, a job
39
+ actually transitioning state, an embedding call actually completing) --
40
+ never a fabricated/synthetic value, per CLAUDE.md's "no fabricated
41
+ metrics" invariant applied to observability data too.
42
+ """
43
+ from __future__ import annotations
44
+
45
+ import logging
46
+ import sys
47
+ from typing import Any
48
+
49
+ import structlog
50
+ from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, Gauge, Histogram, generate_latest
51
+
52
+ REGISTRY = CollectorRegistry()
53
+
54
+ search_requests_total = Counter(
55
+ "search_requests_total", "Total search requests served, by search_type", ["search_type"], registry=REGISTRY
56
+ )
57
+ search_latency_seconds = Histogram(
58
+ "search_latency_seconds", "Search request latency in seconds", registry=REGISTRY
59
+ )
60
+ search_zero_result_total = Counter(
61
+ "search_zero_result_total", "Total search requests that returned zero results", registry=REGISTRY
62
+ )
63
+ indexing_jobs_total = Counter(
64
+ "indexing_jobs_total", "Total indexing jobs, by terminal/transition status", ["status"], registry=REGISTRY
65
+ )
66
+ indexing_duration_seconds = Histogram(
67
+ "indexing_duration_seconds", "Wall-clock duration of one index_directory() run, in seconds", registry=REGISTRY
68
+ )
69
+ files_indexed_total = Counter(
70
+ "files_indexed_total", "Total files processed across all indexing runs", registry=REGISTRY
71
+ )
72
+ symbols_indexed_total = Counter(
73
+ "symbols_indexed_total", "Total symbols processed across all indexing runs", registry=REGISTRY
74
+ )
75
+ embedding_duration_seconds = Histogram(
76
+ "embedding_duration_seconds", "Duration of one CodeEmbedder.encode() call, in seconds", registry=REGISTRY
77
+ )
78
+ job_queue_depth = Gauge(
79
+ "job_queue_depth", "Current number of queued+running IndexJob rows", registry=REGISTRY
80
+ )
81
+ api_errors_total = Counter(
82
+ "api_errors_total", "Total error responses, by endpoint path and status code", ["endpoint", "status_code"], registry=REGISTRY
83
+ )
84
+
85
+
86
+ def render_metrics() -> tuple[bytes, str]:
87
+ """Returns `(body, content_type)` for the `/metrics` endpoint -- the real, current state of `REGISTRY`, generated fresh on every call (never cached/stale)."""
88
+ return generate_latest(REGISTRY), CONTENT_TYPE_LATEST
89
+
90
+
91
+ _logging_configured = False
92
+
93
+
94
+ def configure_logging() -> None:
95
+ """
96
+ Configures `structlog` for JSON-rendered structured log output, per
97
+ §18: "structured (JSON via structlog)... every log line carries
98
+ enough to reconstruct a request's full path." Idempotent -- safe to
99
+ call more than once (e.g. once per test module importing
100
+ `codepulse.api.app`), only actually reconfigures the first time.
101
+ """
102
+ global _logging_configured
103
+ if _logging_configured:
104
+ return
105
+ structlog.configure(
106
+ processors=[
107
+ structlog.processors.TimeStamper(fmt="iso"),
108
+ structlog.processors.add_log_level,
109
+ structlog.processors.StackInfoRenderer(),
110
+ structlog.processors.format_exc_info,
111
+ structlog.processors.JSONRenderer(),
112
+ ],
113
+ logger_factory=structlog.PrintLoggerFactory(file=sys.stdout),
114
+ wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
115
+ cache_logger_on_first_use=True,
116
+ )
117
+ _logging_configured = True
118
+
119
+
120
+ def get_logger() -> Any:
121
+ """
122
+ Returns a structlog bound logger. §18's own exact field set
123
+ (`request_id`, `repository_id`, `job_id`, `api_key_id`,
124
+ `operation`, `duration_ms`, `error_type`) is bound by each CALL
125
+ SITE via `.bind(...)`/keyword arguments -- this function does not
126
+ hardcode which fields a given log line carries, since not every
127
+ operation has all of them (e.g. a search request has no `job_id`).
128
+ """
129
+ configure_logging()
130
+ return structlog.get_logger()
@@ -0,0 +1,8 @@
1
+ """
2
+ CodePulse AST parsing subsystem.
3
+ """
4
+
5
+ from codepulse.parser.base import LanguageParser
6
+ from codepulse.parser.factory import get_parser_for_file, get_parser_for_language
7
+
8
+ __all__ = ["LanguageParser", "get_parser_for_file", "get_parser_for_language"]
@@ -0,0 +1,58 @@
1
+ from __future__ import annotations
2
+
3
+ from abc import ABC, abstractmethod
4
+ from codepulse.models import (
5
+ FileParseResult,
6
+ ImportStatement,
7
+ Symbol,
8
+ CallReference,
9
+ compute_content_hash,
10
+ )
11
+
12
+
13
+ class LanguageParser(ABC):
14
+ """Abstract Base Class for language-specific AST parsers."""
15
+
16
+ @property
17
+ @abstractmethod
18
+ def language(self) -> str:
19
+ """Returns the canonical language identifier (e.g. 'python', 'javascript')."""
20
+ pass
21
+
22
+ @abstractmethod
23
+ def extract_symbols(self, source_code: str, file_path: str = "") -> list[Symbol]:
24
+ """Extract functions, classes, methods, and types with their signatures and docstrings."""
25
+ pass
26
+
27
+ @abstractmethod
28
+ def extract_imports(self, source_code: str, file_path: str = "") -> list[ImportStatement]:
29
+ """Extract module imports and imported identifiers."""
30
+ pass
31
+
32
+ @abstractmethod
33
+ def extract_calls(
34
+ self, source_code: str, file_path: str = "", symbols: list[Symbol] | None = None
35
+ ) -> list[CallReference]:
36
+ """Extract function/method invocations mapped to their enclosing caller symbols."""
37
+ pass
38
+
39
+ def parse_file(self, file_path: str, source_code: str) -> FileParseResult:
40
+ """Parse source code completely and return aggregated FileParseResult."""
41
+ symbols = self.extract_symbols(source_code, file_path=file_path)
42
+ imports = self.extract_imports(source_code, file_path=file_path)
43
+ calls = self.extract_calls(source_code, file_path=file_path, symbols=symbols)
44
+
45
+ line_count = len(source_code.splitlines())
46
+ file_size_bytes = len(source_code.encode("utf-8"))
47
+ content_hash = compute_content_hash(source_code)
48
+
49
+ return FileParseResult(
50
+ file_path=file_path,
51
+ language=self.language,
52
+ symbols=symbols,
53
+ imports=imports,
54
+ calls=calls,
55
+ file_size_bytes=file_size_bytes,
56
+ line_count=line_count,
57
+ content_hash=content_hash,
58
+ )
@@ -0,0 +1,37 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from codepulse.parser.base import LanguageParser
5
+ from codepulse.parser.python import PythonParser
6
+ from codepulse.parser.javascript import JavaScriptParser
7
+
8
+ _PARSERS: dict[str, LanguageParser] = {}
9
+
10
+
11
+ def get_parser_for_language(language: str) -> LanguageParser:
12
+ """Retrieve a cached parser instance for the requested language."""
13
+ lang_lower = language.lower().strip()
14
+ if lang_lower not in _PARSERS:
15
+ if lang_lower in ("python", "py"):
16
+ _PARSERS[lang_lower] = PythonParser()
17
+ elif lang_lower in ("javascript", "js"):
18
+ _PARSERS[lang_lower] = JavaScriptParser(is_typescript=False)
19
+ elif lang_lower in ("typescript", "ts"):
20
+ _PARSERS[lang_lower] = JavaScriptParser(is_typescript=True)
21
+ else:
22
+ raise ValueError(f"Unsupported language: '{language}'")
23
+ return _PARSERS[lang_lower]
24
+
25
+
26
+ def get_parser_for_file(file_path: str) -> LanguageParser | None:
27
+ """Determine the appropriate parser based on file extension."""
28
+ _, ext = os.path.splitext(file_path)
29
+ ext_lower = ext.lower()
30
+
31
+ if ext_lower == ".py":
32
+ return get_parser_for_language("python")
33
+ elif ext_lower in (".js", ".jsx", ".mjs", ".cjs"):
34
+ return get_parser_for_language("javascript")
35
+ elif ext_lower in (".ts", ".tsx", ".mts", ".cts"):
36
+ return get_parser_for_language("typescript")
37
+ return None