codepulse-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. codepulse/__init__.py +5 -0
  2. codepulse/api/app.py +1336 -0
  3. codepulse/api/errors.py +172 -0
  4. codepulse/api/security.py +189 -0
  5. codepulse/auth.py +406 -0
  6. codepulse/cli.py +1052 -0
  7. codepulse/config.py +475 -0
  8. codepulse/data_location.py +78 -0
  9. codepulse/embeddings/chunker.py +138 -0
  10. codepulse/embeddings/embedder.py +120 -0
  11. codepulse/git_diff.py +201 -0
  12. codepulse/indexer/engine.py +2655 -0
  13. codepulse/indexer/ignore.py +120 -0
  14. codepulse/ingestion_safety.py +169 -0
  15. codepulse/jobs.py +257 -0
  16. codepulse/mcp_server.py +269 -0
  17. codepulse/models.py +140 -0
  18. codepulse/observability.py +130 -0
  19. codepulse/parser/__init__.py +8 -0
  20. codepulse/parser/base.py +58 -0
  21. codepulse/parser/factory.py +37 -0
  22. codepulse/parser/graph.py +502 -0
  23. codepulse/parser/javascript.py +428 -0
  24. codepulse/parser/module_graph.py +232 -0
  25. codepulse/parser/python.py +331 -0
  26. codepulse/rate_limit.py +142 -0
  27. codepulse/repository_identity.py +83 -0
  28. codepulse/search/bm25.py +342 -0
  29. codepulse/search/ranker.py +382 -0
  30. codepulse/search/tokenizer.py +80 -0
  31. codepulse/search/vector.py +357 -0
  32. codepulse/storage/db.py +526 -0
  33. codepulse/storage/migrations.py +347 -0
  34. codepulse/storage/schema_migrations/README +1 -0
  35. codepulse/storage/schema_migrations/env.py +70 -0
  36. codepulse/storage/schema_migrations/script.py.mako +28 -0
  37. codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
  38. codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
  39. codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
  40. codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
  41. codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
  42. codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
  43. codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
  44. codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
  45. codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
  46. codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
  47. codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
  48. codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
  49. codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
  50. codepulse/symbol_identity.py +29 -0
  51. codepulse_engine-0.1.0.dist-info/METADATA +191 -0
  52. codepulse_engine-0.1.0.dist-info/RECORD +56 -0
  53. codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
  54. codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
  55. codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
  56. codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,526 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from datetime import datetime
5
+ from typing import Optional
6
+ from sqlalchemy import (
7
+ Column,
8
+ Integer,
9
+ BigInteger,
10
+ String,
11
+ Text,
12
+ DateTime,
13
+ ForeignKey,
14
+ Float,
15
+ UniqueConstraint,
16
+ create_engine,
17
+ event,
18
+ inspect,
19
+ )
20
+ from sqlalchemy.orm import declarative_base, relationship, sessionmaker, Session
21
+
22
+ Base = declarative_base()
23
+
24
+
25
+ class RepositoryModel(Base):
26
+ __tablename__ = "repositories"
27
+ __table_args__ = (
28
+ # Natural key for repository identity (Phase 0.3): a repository is
29
+ # identified by its `url` (the git URL, or the absolute local path
30
+ # when indexing a local directory with no explicit URL -- see
31
+ # IndexingEngine.index_directory). Enforced at the DB level so a
32
+ # duplicate Repository row can never be created for the same
33
+ # logical repository, even under a bug or a race.
34
+ UniqueConstraint("url", name="uq_repositories_url"),
35
+ )
36
+
37
+ id = Column(Integer, primary_key=True, autoincrement=True)
38
+ team_id = Column(Integer, default=1, nullable=False)
39
+ name = Column(String(255), nullable=False)
40
+ url = Column(Text, nullable=False)
41
+ branch = Column(String(100), default="main")
42
+ last_sync = Column(DateTime, default=datetime.utcnow)
43
+ file_count = Column(Integer, default=0)
44
+ symbol_count = Column(Integer, default=0)
45
+
46
+ files = relationship("FileModel", back_populates="repository", cascade="all, delete-orphan")
47
+
48
+
49
+ class FileModel(Base):
50
+ __tablename__ = "files"
51
+ __table_args__ = (
52
+ # Natural key for file identity (Phase 0.3): a file is identified by
53
+ # (repo_id, path). Enforced at the DB level so a duplicate File row
54
+ # can never be created for the same path within the same repository.
55
+ UniqueConstraint("repo_id", "path", name="uq_files_repo_id_path"),
56
+ )
57
+
58
+ id = Column(Integer, primary_key=True, autoincrement=True)
59
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False)
60
+ path = Column(Text, nullable=False)
61
+ language = Column(String(50), nullable=False)
62
+ size_bytes = Column(Integer, default=0)
63
+ line_count = Column(Integer, default=0)
64
+ # SHA-256 hex digest of the file's content (see codepulse.models.compute_content_hash).
65
+ # Nullable because historical rows created before Phase 0.3 have no recorded
66
+ # original content to hash (see the Phase 0.3 migration for details) --
67
+ # such rows get a real hash the next time the file is (re-)indexed.
68
+ content_hash = Column(String(64), nullable=True)
69
+ last_indexed = Column(DateTime, default=datetime.utcnow)
70
+
71
+ repository = relationship("RepositoryModel", back_populates="files")
72
+ symbols = relationship("SymbolModel", back_populates="file", cascade="all, delete-orphan")
73
+
74
+
75
+ class SymbolModel(Base):
76
+ __tablename__ = "symbols"
77
+ __table_args__ = (
78
+ # Natural key for symbol identity (Phase 0.4): a symbol is identified
79
+ # by (file_id, name, start_line), per the roadmap's Phase 0
80
+ # idempotency rule. Enforced at the DB level so a duplicate Symbol
81
+ # row can never be created for the same symbol within the same file.
82
+ UniqueConstraint(
83
+ "file_id", "name", "start_line", name="uq_symbols_file_id_name_start_line"
84
+ ),
85
+ )
86
+
87
+ id = Column(Integer, primary_key=True, autoincrement=True)
88
+ file_id = Column(Integer, ForeignKey("files.id", ondelete="CASCADE"), nullable=False)
89
+ name = Column(String(255), nullable=False, index=True)
90
+ type = Column(String(50), nullable=False)
91
+ start_line = Column(Integer, nullable=False)
92
+ end_line = Column(Integer, nullable=False)
93
+ signature = Column(Text, default="")
94
+ docstring = Column(Text, nullable=True)
95
+ body = Column(Text, default="")
96
+ body_hash = Column(String(64), default="")
97
+ language = Column(String(50), nullable=False)
98
+ complexity = Column(Integer, default=1)
99
+ # Phase 4.1 (roadmap §37 item 4.1; §29: "Symbol... needs... base-class
100
+ # field"). JSON-serialized `list[str]` of raw, unresolved base-class
101
+ # expressions (e.g. '["Base", "pkg.Mixin"]') -- the same "Text column
102
+ # holding a JSON-serialized list, parsed in application code"
103
+ # convention already established by `EmbeddingModel.embedding_json`,
104
+ # not a new serialization approach. `NULL` for a never-(re)indexed-
105
+ # since-this-migration row (see that migration's own docstring) and
106
+ # for every non-CLASS symbol; application code treats `NULL` the same
107
+ # as an empty list, never as "unknown."
108
+ base_classes = Column(Text, nullable=True)
109
+
110
+ file = relationship("FileModel", back_populates="symbols")
111
+ embeddings = relationship("EmbeddingModel", back_populates="symbol", cascade="all, delete-orphan")
112
+
113
+
114
+ class CallEdge(Base):
115
+ """
116
+ Phase 3.4 (roadmap §37 item 3.4: "CallEdge.confidence scoring"; §11:
117
+ "every `CallEdge` should carry a `confidence` field -- `high` when
118
+ resolved through an explicit import mapping to a known file+symbol,
119
+ `low` when resolved only by bare-name match against same-file
120
+ symbols, `unresolved` when the callee couldn't be matched to any
121
+ known symbol at all"). Renamed from `DependencyModel` (table
122
+ `dependencies` -> `call_edges`) per §29's Data Model target entity
123
+ name, migration `f6e481353553`.
124
+
125
+ `confidence` is nullable: rows persisted before this migration have
126
+ no historical resolution-path information to backfill honestly
127
+ (CLAUDE.md: no fabricated metrics) and show `NULL` until their
128
+ repository's next re-index recomputes and persists a real value --
129
+ see that migration's own docstring.
130
+
131
+ RESOLVED by Phase 3.5 ("local BFS-based incremental graph
132
+ updates"): rows are no longer blindly appended every run. A full
133
+ run replaces every row for the repository; an incremental run
134
+ deletes and recomputes only the rows whose source symbol changed
135
+ this run, plus (the "local BFS one hop out") any OTHER symbol's
136
+ row whose target was a symbol that changed, was renamed, or
137
+ disappeared -- see `IndexingEngine._persist_call_graph_edges`'s own
138
+ docstring for the full algorithm. Before Phase 3.5 (and under the
139
+ `DependencyModel` name before that), rows simply accumulated
140
+ forever every run -- a long-disclosed Phase 0.4 limitation, now
141
+ closed.
142
+
143
+ `repo_id` (post-Phase-3.4 isolation fix, migration `2cd38eb12a7e`):
144
+ every row written by `IndexingEngine.
145
+ _persist_to_db` from here on carries the repository it belongs to.
146
+ Before this fix, `CallEdge` (and `DependencyModel` before it) had NO
147
+ repository association at all -- a confirmed, real cross-repository
148
+ data leak: `GET /v1/symbols/{id}` queried this table by symbol name
149
+ alone, so a caller authorized for one repository could receive
150
+ another, unrelated repository's call-graph edges whenever symbol
151
+ names happened to collide. Nullable because some historical rows
152
+ (persisted before this fix existed) cannot be attributed to a
153
+ repository with full confidence -- see that migration's own
154
+ docstring for the exact backfill rule (unambiguous match only, never
155
+ a guess). A `NULL`-`repo_id` row is invisible to every
156
+ repository-scoped query by construction (a real, honestly-disclosed
157
+ completeness gap for that specific historical edge, not a security
158
+ problem) until its repository is next re-indexed, which persists a
159
+ fresh, correctly-scoped replacement.
160
+
161
+ `relationship_type` (Phase 4.1): `"calls"` (the only kind this table
162
+ has ever held, and the default/backfill value for every row that
163
+ predates this column -- unambiguous, not a guess, since nothing else
164
+ was ever written here) or `"inherits"` (a resolved base-class
165
+ relationship, roadmap §37 item 4.1). Both kinds share this one
166
+ table -- same `repo_id` scoping, same `confidence` vocabulary, same
167
+ Phase 3.5 incremental-update machinery -- but are kept in SEPARATE
168
+ in-memory adjacency structures by `CallGraphBuilder`
169
+ (`self.adjacency`/`self.reverse_adjacency` for `"calls"` only; a
170
+ distinct pair for `"inherits"`), so `get_callers`/`get_callees`
171
+ continue to traverse call relationships only, exactly as before this
172
+ column existed -- see `CallGraphBuilder._build_graph`'s own
173
+ docstring.
174
+ """
175
+ __tablename__ = "call_edges"
176
+
177
+ id = Column(Integer, primary_key=True, autoincrement=True)
178
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=True, index=True)
179
+ source_symbol_name = Column(String(255), nullable=False, index=True)
180
+ source_file = Column(Text, nullable=False)
181
+ target_symbol_name = Column(String(255), nullable=False, index=True)
182
+ target_file = Column(Text, nullable=True)
183
+ call_count = Column(Integer, default=1)
184
+ relationship_type = Column(String(20), nullable=False, default="calls", server_default="calls")
185
+ confidence = Column(String(20), nullable=True)
186
+
187
+
188
+ class ModuleEdge(Base):
189
+ """
190
+ The persisted file-level import graph of one repository, so a restarted
191
+ process can rebuild module structure without re-parsing -- the graph
192
+ used to exist only in `IndexingEngine.parse_results` (in memory), so
193
+ module structure was empty after every restart
194
+ (CODEPULSE_FUTURE_CHANGES.md item 15). Migration `7c3e5a91d2f4`.
195
+
196
+ One row per RESOLVED edge (`source_file` imports `target_file`, both
197
+ repository-local; an external or stdlib import never becomes an edge,
198
+ per `ModuleGraphBuilder`'s own rules). A file that has no resolved
199
+ imports gets one row with `target_file` NULL: the node set is stored
200
+ explicitly, so (a) isolated files are kept, (b) the restored graph has
201
+ exactly the files that were parsed (not whatever `files` rows exist,
202
+ which can include stale rows), and (c) "no rows for a repository"
203
+ reliably means "not indexed since this table existed" -- the repository
204
+ stays cold instead of showing a misleading all-singletons graph.
205
+
206
+ Rows are replaced wholesale for the repository on every index run (full
207
+ or incremental) from the complete merged parse set, so they are always
208
+ a coherent snapshot and re-indexing never duplicates them. `repo_id` is
209
+ NOT NULL and cascades on repository deletion.
210
+ """
211
+ __tablename__ = "module_edges"
212
+ __table_args__ = (
213
+ UniqueConstraint("repo_id", "source_file", "target_file", name="uq_module_edges_repo_source_target"),
214
+ )
215
+
216
+ id = Column(Integer, primary_key=True, autoincrement=True)
217
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
218
+ source_file = Column(Text, nullable=False)
219
+ target_file = Column(Text, nullable=True)
220
+
221
+
222
+ class EmbeddingModel(Base):
223
+ __tablename__ = "code_embeddings"
224
+ __table_args__ = (
225
+ # Natural key for embedding identity (Phase 0.5), per the roadmap's
226
+ # Phase 0 idempotency rule (§8): "(symbol_id, chunk_type,
227
+ # chunk_index)". Enforced at the DB level so a duplicate Embedding
228
+ # row can never be created for the same chunk of the same symbol.
229
+ UniqueConstraint(
230
+ "symbol_id", "chunk_type", "chunk_index", name="uq_code_embeddings_symbol_chunk"
231
+ ),
232
+ )
233
+
234
+ id = Column(Integer, primary_key=True, autoincrement=True)
235
+ symbol_id = Column(Integer, ForeignKey("symbols.id", ondelete="CASCADE"), nullable=False)
236
+ chunk_type = Column(String(50), default="signature_doc") # 'signature_doc', 'body_chunk', 'full'
237
+ # Position of this chunk among a symbol's chunks (see
238
+ # codepulse.embeddings.chunker.CodeChunk.chunk_index); part of the
239
+ # natural key together with (symbol_id, chunk_type).
240
+ chunk_index = Column(Integer, nullable=False, default=0)
241
+ embedding_json = Column(Text, nullable=False) # JSON serialized array of floats for portability
242
+ chunk_content = Column(Text, default="")
243
+ # SHA-256 hex digest of `chunk_content` (see codepulse.models.compute_content_hash),
244
+ # used for change detection: reuse the row if unchanged, update in place
245
+ # if the chunk's source content changed. Nullable because historical
246
+ # rows created before Phase 0.5 have no recorded hash.
247
+ content_hash = Column(String(64), nullable=True)
248
+ # Identity of the model that produced this vector, and its dimension --
249
+ # recorded so hydration can detect a model/dimension change and skip
250
+ # loading an incompatible vector rather than silently corrupting the
251
+ # in-memory vector index (roadmap §9: "embedding dimension/model must be
252
+ # recorded"). Nullable for the same reason as content_hash.
253
+ embedding_model = Column(String(150), nullable=True)
254
+ embedding_dim = Column(Integer, nullable=True)
255
+
256
+ symbol = relationship("SymbolModel", back_populates="embeddings")
257
+
258
+
259
+ class APIKeyModel(Base):
260
+ """
261
+ Phase 1.1 (roadmap §15, §37 item 1.1): API key model + hashing +
262
+ creation flow.
263
+
264
+ The raw key is NEVER stored -- only its SHA-256 hash (`key_hash`, see
265
+ codepulse.auth.hash_api_key) and a short, non-secret prefix
266
+ (`key_prefix`) kept purely for display/identification in a future
267
+ key-management UI/CLI. This is Phase 1.1 scope only: the model,
268
+ hashing, and a creation (+ verification-primitive) flow. Enforcing
269
+ this at the API boundary (a FastAPI dependency required on every
270
+ route) is Phase 1.2, deliberately not implemented here. Role-based
271
+ permissions (viewer/indexer/admin, roadmap §15) were Phase 7 (B25)
272
+ scope -- see `APIKeyRepositoryModel.role`'s own docstring; the role
273
+ lives on the per-repository grant row, not here on `APIKeyModel`
274
+ itself, per §15's own "enforced at the repository-permission level,
275
+ not globally."
276
+ """
277
+ __tablename__ = "api_keys"
278
+ __table_args__ = (
279
+ UniqueConstraint("key_hash", name="uq_api_keys_key_hash"),
280
+ )
281
+
282
+ id = Column(Integer, primary_key=True, autoincrement=True)
283
+ name = Column(String(255), nullable=False)
284
+ # SHA-256 hex digest of the raw key (codepulse.auth.hash_api_key). A
285
+ # fast cryptographic hash (not a slow/salted password hash) is
286
+ # appropriate here because the input is already a high-entropy,
287
+ # randomly generated secret (codepulse.auth.generate_raw_api_key), not
288
+ # a user-chosen low-entropy password -- and a fast hash keeps
289
+ # lookup-by-hash a simple indexed equality query for Phase 1.2's
290
+ # authorization dependency.
291
+ key_hash = Column(String(64), nullable=False)
292
+ # Short, non-secret prefix of the raw key (e.g. "cpk_a1b2c3d4"), kept
293
+ # only so a key can be identified/distinguished in a future listing
294
+ # UI/CLI without ever re-displaying the secret itself.
295
+ key_prefix = Column(String(32), nullable=False)
296
+ created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
297
+ # Nullable revocation timestamp: schema support only in Phase 1.1 (so
298
+ # a later revocation milestone doesn't need its own migration) -- no
299
+ # revoke() flow is implemented yet; codepulse.auth.verify_api_key does
300
+ # respect this field defensively (a revoked key never verifies).
301
+ revoked_at = Column(DateTime, nullable=True)
302
+
303
+ repositories = relationship(
304
+ "APIKeyRepositoryModel", back_populates="api_key", cascade="all, delete-orphan"
305
+ )
306
+
307
+
308
+ class APIKeyRepositoryModel(Base):
309
+ """
310
+ Scopes an APIKey to a specific repository (roadmap §15: "API keys
311
+ (hashed at rest, scoped to one or more repositories)"). One row per
312
+ (api_key, repository) pair a key is permitted to access. This table
313
+ only records the grant -- Phase 1.1 does not read or enforce it
314
+ anywhere; that's Phase 1.2's central authorization dependency.
315
+
316
+ Phase 7.2 (roadmap §37 item 7.2 "RBAC roles + repo permission
317
+ model"; §15's own literal minimal-viable model: *"roles `viewer`
318
+ (search/read), `indexer` (viewer + trigger indexing), `admin` (all +
319
+ manage keys/repos). Enforced at the repository-permission level, not
320
+ globally."*): `role` lives HERE, on the per-(key, repository) grant
321
+ row, not on `APIKeyModel` itself -- exactly matching "not globally."
322
+ A single key can legitimately be `viewer` on one repository and
323
+ `admin` on another.
324
+
325
+ `NOT NULL DEFAULT 'admin'`: every grant created before this
326
+ milestone had, in practice, UNRESTRICTED access to every endpoint
327
+ that checked repository membership alone (no role concept existed
328
+ to restrict it) -- backfilling to `'admin'` reflects each existing
329
+ grant's actual historical capability honestly, not a guess (unlike
330
+ an ambiguous backfill elsewhere in this project's history, this one
331
+ has a single, unambiguous correct answer). `codepulse.auth.
332
+ create_api_key`'s own default for NEWLY created grants going
333
+ forward is intentionally the safer `'viewer'` (least privilege),
334
+ matching this table's forward-looking correctness rather than its
335
+ backfilled historical value.
336
+ """
337
+ __tablename__ = "api_key_repositories"
338
+ __table_args__ = (
339
+ UniqueConstraint("api_key_id", "repo_id", name="uq_api_key_repositories_key_repo"),
340
+ )
341
+
342
+ id = Column(Integer, primary_key=True, autoincrement=True)
343
+ api_key_id = Column(Integer, ForeignKey("api_keys.id", ondelete="CASCADE"), nullable=False)
344
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False)
345
+ role = Column(String(20), nullable=False, default="admin", server_default="admin")
346
+
347
+ api_key = relationship("APIKeyModel", back_populates="repositories")
348
+
349
+
350
+ class IndexJobModel(Base):
351
+ """
352
+ Phase 1.3 (roadmap §37 item 1.3; §8 "Concurrency/failure handling";
353
+ §29 Data Model; backlog B9). Tracks one indexing run for a repository
354
+ through the state machine roadmap §8 describes literally: `queued ->
355
+ running -> succeeded|failed|cancelled`. Replaces the previous fake
356
+ `job_id` (`f"job_{int(time.time())}"`, no backing state at all) that
357
+ `/v1/repositories/index` used to return -- B9's DoD: "Returned job_id
358
+ is pollable to real completion" (see `GET /v1/jobs/{job_id}`).
359
+
360
+ Unlike every other model in this file, this table is NOT an
361
+ upsert-by-natural-key entity (no `UniqueConstraint` here) -- a job is a
362
+ record of one indexing *run* (an event), not an idempotent identity to
363
+ be reused across runs; every indexing attempt for a repository gets
364
+ its own row, by design, forming a run history. This does not weaken
365
+ the project's "idempotent indexing" invariant, which governs the
366
+ File/Symbol/Embedding rows an indexing run itself writes -- unaffected
367
+ by this table.
368
+
369
+ `cancelled` is a valid, permanent schema value (so a later
370
+ cancellation milestone doesn't need its own migration) but is never
371
+ actually written by this milestone -- no cancel flow/endpoint exists
372
+ yet, matching how Phase 1.1 left `APIKeyModel.revoked_at` schema-only
373
+ until a revoke() flow existed.
374
+
375
+ Row lifecycle logic (create / transition / query) lives in
376
+ `codepulse.jobs`, not here and not in `IndexingEngine` -- this table
377
+ tracks *orchestration* state for one indexing run, the same
378
+ boundary-level category as `APIKeyModel`/`AuditEvent` in the roadmap's
379
+ own §29 Data Model table, not a core intelligence entity.
380
+ """
381
+ __tablename__ = "index_jobs"
382
+
383
+ id = Column(Integer, primary_key=True, autoincrement=True)
384
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
385
+ status = Column(String(20), nullable=False, default="queued")
386
+ created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
387
+ started_at = Column(DateTime, nullable=True)
388
+ finished_at = Column(DateTime, nullable=True)
389
+ error_message = Column(Text, nullable=True)
390
+ # JSON-serialized copy of IndexingEngine.index_directory()'s real
391
+ # return dict (repository_name, total_files, total_symbols, ... -- see
392
+ # engine.py) once the job succeeds. Never fabricated/estimated --
393
+ # populated only from the actual measured result, per this project's
394
+ # "no fabricated metrics" invariant.
395
+ result_summary = Column(Text, nullable=True)
396
+
397
+ repository = relationship("RepositoryModel")
398
+
399
+
400
+ class RepositorySnapshotModel(Base):
401
+ """
402
+ Phase 3.1 (roadmap §37 item 3.1: "Git-diff change detection
403
+ (added/modified/deleted)"; §9 Incremental Indexing Design; §29 Data
404
+ Model, which explicitly names `RepositorySnapshot` as a required-now
405
+ entity: "Required for incremental indexing (§9) and crash-safe reads
406
+ (§17)"). Records, for one successful indexing run, the git commit
407
+ the repository's content was at (`commit_sha`, `None` for a
408
+ non-git/local-path repository -- git-diff-based change detection is
409
+ only possible for a real git repository) and the embedding model/
410
+ dimension used -- both are exactly the two pieces of state §9's own
411
+ "when a full re-index is still required" list needs to check on the
412
+ *next* run: "if no prior successful snapshot exists, [or] if the
413
+ embedding model version changed".
414
+
415
+ Like `IndexJobModel`, this is a run-history table (no natural-key
416
+ `UniqueConstraint`) -- one row per successful `index_directory()`
417
+ call, not an upsert-by-identity entity. `IndexingEngine._plan_
418
+ incremental_run` reads the single most recent row for a `repo_id` to
419
+ decide whether an incremental (git-diff-driven) run is possible; a
420
+ full re-index still creates its own new row here exactly the same
421
+ way, so "no prior successful snapshot" naturally self-heals after
422
+ the very first successful run.
423
+
424
+ This milestone does NOT implement §17's full crash-safe,
425
+ status-gated "always serve the latest ready snapshot" read semantics
426
+ -- that is a broader reliability feature §29 attributes to this same
427
+ entity for a *later* purpose, not something Phase 3.1's own literal
428
+ "git-diff change detection" scope requires building now. No `status`
429
+ column exists here for that reason; disclosed, not silently
430
+ expanded into.
431
+ """
432
+ __tablename__ = "repository_snapshots"
433
+
434
+ id = Column(Integer, primary_key=True, autoincrement=True)
435
+ repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
436
+ commit_sha = Column(String(64), nullable=True)
437
+ embedding_model = Column(String(255), nullable=True)
438
+ embedding_dim = Column(Integer, nullable=True)
439
+ indexed_at = Column(DateTime, default=datetime.utcnow, nullable=False)
440
+
441
+ repository = relationship("RepositoryModel")
442
+
443
+
444
+ class StorageManager:
445
+ """Manages database connection, tables, and session lifecycle."""
446
+
447
+ def __init__(self, db_url: str | None = None) -> None:
448
+ if db_url is None:
449
+ from codepulse.config import get_settings
450
+ db_url = get_settings().db_url
451
+ self.db_url = db_url
452
+ # The per-user data directory is created on demand; any other missing directory is still an error.
453
+ from codepulse.data_location import ensure_database_directory
454
+
455
+ ensure_database_directory(db_url)
456
+ self.engine = create_engine(db_url, echo=False)
457
+
458
+ # Backend-hardening addition (not a roadmap sprint item): SQLite
459
+ # disables foreign-key enforcement by default, per-connection --
460
+ # every ON DELETE CASCADE declared on a ForeignKey() above (e.g.
461
+ # CallEdge.repo_id, APIKeyRepositoryModel.api_key_id/repo_id,
462
+ # IndexJobModel.repo_id, RepositorySnapshotModel.repo_id) was
463
+ # previously inert, a real, disclosed gap (see CODEPULSE_PROGRESS.md's
464
+ # "Known technical debt"). Scoped to THIS engine instance only
465
+ # (not the global sqlalchemy.engine.Engine class) so it applies to
466
+ # every connection the application makes through a StorageManager,
467
+ # while deliberately NOT touching Alembic's own, entirely separate
468
+ # `engine_from_config(...)` connection (alembic/env.py) -- migration
469
+ # execution is unaffected by this change. Guarded to SQLite only,
470
+ # since PRAGMA is SQLite-specific syntax and Postgres (Phase 7.1,
471
+ # not implemented) enforces foreign keys by default already, no
472
+ # pragma needed.
473
+ @event.listens_for(self.engine, "connect")
474
+ def _enable_sqlite_foreign_keys(dbapi_connection, connection_record) -> None:
475
+ if self.engine.dialect.name != "sqlite":
476
+ return
477
+ cursor = dbapi_connection.cursor()
478
+ cursor.execute("PRAGMA foreign_keys=ON")
479
+ cursor.close()
480
+
481
+ self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self.engine)
482
+ self.create_tables()
483
+
484
+ def create_tables(self) -> None:
485
+ """
486
+ Creates every table declared on `Base.metadata` that doesn't yet
487
+ exist -- but ONLY on a database Alembic has never touched (no
488
+ `alembic_version` table present). Backend-hardening fix, closing
489
+ the actual root cause of three separate, real, documented
490
+ incidents (see CODEPULSE_PROGRESS.md's "Known technical debt"):
491
+ an ad-hoc `StorageManager()` construction against the REAL,
492
+ already-migrated `codepulse.db` (e.g. importing
493
+ `codepulse.api.app` without an isolated `CODEPULSE_DB_URL`)
494
+ could silently `create_all()` a table a pending Alembic
495
+ migration hadn't created yet, racing ahead of Alembic and
496
+ leaving `alembic_version` stale -- requiring manual
497
+ remediation each time it happened.
498
+
499
+ A fresh, disposable database (every test fixture in this
500
+ project's own suite, e.g. `tests/conftest.py`'s `isolated_app`)
501
+ has no `alembic_version` table yet, so this remains a true
502
+ zero-setup `create_all()` for that case, unchanged from before
503
+ this fix -- `tests/test_migrations.py::
504
+ test_existing_database_stamping_scenario_2` exercises exactly
505
+ this "no alembic_version yet" precondition directly. The real
506
+ `codepulse.db` has had an `alembic_version` table since its very
507
+ first migration; from that point on, this method is a safe
508
+ no-op for it, and schema changes must go through Alembic only.
509
+ """
510
+ inspector = inspect(self.engine)
511
+ if inspector.has_table("alembic_version"):
512
+ return
513
+ brand_new = not inspector.get_table_names()
514
+ Base.metadata.create_all(bind=self.engine)
515
+ if brand_new:
516
+ # A database created from scratch here IS the current schema, so record that. Until
517
+ # 2026-10-14 such databases were never tracked by Alembic and could not be upgraded
518
+ # later (FUTURE_CHANGES items 8/17). A database that already had tables but no
519
+ # alembic_version (created by older code) is deliberately NOT stamped here: whether
520
+ # its schema matches is what `codepulse migrate` checks before it stamps.
521
+ from codepulse.storage.migrations import stamp_new_database
522
+
523
+ stamp_new_database(self.engine)
524
+
525
+ def get_session(self) -> Session:
526
+ return self.SessionLocal()