codepulse-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codepulse/__init__.py +5 -0
- codepulse/api/app.py +1336 -0
- codepulse/api/errors.py +172 -0
- codepulse/api/security.py +189 -0
- codepulse/auth.py +406 -0
- codepulse/cli.py +1052 -0
- codepulse/config.py +475 -0
- codepulse/data_location.py +78 -0
- codepulse/embeddings/chunker.py +138 -0
- codepulse/embeddings/embedder.py +120 -0
- codepulse/git_diff.py +201 -0
- codepulse/indexer/engine.py +2655 -0
- codepulse/indexer/ignore.py +120 -0
- codepulse/ingestion_safety.py +169 -0
- codepulse/jobs.py +257 -0
- codepulse/mcp_server.py +269 -0
- codepulse/models.py +140 -0
- codepulse/observability.py +130 -0
- codepulse/parser/__init__.py +8 -0
- codepulse/parser/base.py +58 -0
- codepulse/parser/factory.py +37 -0
- codepulse/parser/graph.py +502 -0
- codepulse/parser/javascript.py +428 -0
- codepulse/parser/module_graph.py +232 -0
- codepulse/parser/python.py +331 -0
- codepulse/rate_limit.py +142 -0
- codepulse/repository_identity.py +83 -0
- codepulse/search/bm25.py +342 -0
- codepulse/search/ranker.py +382 -0
- codepulse/search/tokenizer.py +80 -0
- codepulse/search/vector.py +357 -0
- codepulse/storage/db.py +526 -0
- codepulse/storage/migrations.py +347 -0
- codepulse/storage/schema_migrations/README +1 -0
- codepulse/storage/schema_migrations/env.py +70 -0
- codepulse/storage/schema_migrations/script.py.mako +28 -0
- codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
- codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
- codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
- codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
- codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
- codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
- codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
- codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
- codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
- codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
- codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
- codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
- codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
- codepulse/symbol_identity.py +29 -0
- codepulse_engine-0.1.0.dist-info/METADATA +191 -0
- codepulse_engine-0.1.0.dist-info/RECORD +56 -0
- codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
- codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
- codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
- codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
codepulse/storage/db.py
ADDED
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Optional
|
|
6
|
+
from sqlalchemy import (
|
|
7
|
+
Column,
|
|
8
|
+
Integer,
|
|
9
|
+
BigInteger,
|
|
10
|
+
String,
|
|
11
|
+
Text,
|
|
12
|
+
DateTime,
|
|
13
|
+
ForeignKey,
|
|
14
|
+
Float,
|
|
15
|
+
UniqueConstraint,
|
|
16
|
+
create_engine,
|
|
17
|
+
event,
|
|
18
|
+
inspect,
|
|
19
|
+
)
|
|
20
|
+
from sqlalchemy.orm import declarative_base, relationship, sessionmaker, Session
|
|
21
|
+
|
|
22
|
+
Base = declarative_base()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RepositoryModel(Base):
|
|
26
|
+
__tablename__ = "repositories"
|
|
27
|
+
__table_args__ = (
|
|
28
|
+
# Natural key for repository identity (Phase 0.3): a repository is
|
|
29
|
+
# identified by its `url` (the git URL, or the absolute local path
|
|
30
|
+
# when indexing a local directory with no explicit URL -- see
|
|
31
|
+
# IndexingEngine.index_directory). Enforced at the DB level so a
|
|
32
|
+
# duplicate Repository row can never be created for the same
|
|
33
|
+
# logical repository, even under a bug or a race.
|
|
34
|
+
UniqueConstraint("url", name="uq_repositories_url"),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
38
|
+
team_id = Column(Integer, default=1, nullable=False)
|
|
39
|
+
name = Column(String(255), nullable=False)
|
|
40
|
+
url = Column(Text, nullable=False)
|
|
41
|
+
branch = Column(String(100), default="main")
|
|
42
|
+
last_sync = Column(DateTime, default=datetime.utcnow)
|
|
43
|
+
file_count = Column(Integer, default=0)
|
|
44
|
+
symbol_count = Column(Integer, default=0)
|
|
45
|
+
|
|
46
|
+
files = relationship("FileModel", back_populates="repository", cascade="all, delete-orphan")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class FileModel(Base):
|
|
50
|
+
__tablename__ = "files"
|
|
51
|
+
__table_args__ = (
|
|
52
|
+
# Natural key for file identity (Phase 0.3): a file is identified by
|
|
53
|
+
# (repo_id, path). Enforced at the DB level so a duplicate File row
|
|
54
|
+
# can never be created for the same path within the same repository.
|
|
55
|
+
UniqueConstraint("repo_id", "path", name="uq_files_repo_id_path"),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
59
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False)
|
|
60
|
+
path = Column(Text, nullable=False)
|
|
61
|
+
language = Column(String(50), nullable=False)
|
|
62
|
+
size_bytes = Column(Integer, default=0)
|
|
63
|
+
line_count = Column(Integer, default=0)
|
|
64
|
+
# SHA-256 hex digest of the file's content (see codepulse.models.compute_content_hash).
|
|
65
|
+
# Nullable because historical rows created before Phase 0.3 have no recorded
|
|
66
|
+
# original content to hash (see the Phase 0.3 migration for details) --
|
|
67
|
+
# such rows get a real hash the next time the file is (re-)indexed.
|
|
68
|
+
content_hash = Column(String(64), nullable=True)
|
|
69
|
+
last_indexed = Column(DateTime, default=datetime.utcnow)
|
|
70
|
+
|
|
71
|
+
repository = relationship("RepositoryModel", back_populates="files")
|
|
72
|
+
symbols = relationship("SymbolModel", back_populates="file", cascade="all, delete-orphan")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class SymbolModel(Base):
|
|
76
|
+
__tablename__ = "symbols"
|
|
77
|
+
__table_args__ = (
|
|
78
|
+
# Natural key for symbol identity (Phase 0.4): a symbol is identified
|
|
79
|
+
# by (file_id, name, start_line), per the roadmap's Phase 0
|
|
80
|
+
# idempotency rule. Enforced at the DB level so a duplicate Symbol
|
|
81
|
+
# row can never be created for the same symbol within the same file.
|
|
82
|
+
UniqueConstraint(
|
|
83
|
+
"file_id", "name", "start_line", name="uq_symbols_file_id_name_start_line"
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
88
|
+
file_id = Column(Integer, ForeignKey("files.id", ondelete="CASCADE"), nullable=False)
|
|
89
|
+
name = Column(String(255), nullable=False, index=True)
|
|
90
|
+
type = Column(String(50), nullable=False)
|
|
91
|
+
start_line = Column(Integer, nullable=False)
|
|
92
|
+
end_line = Column(Integer, nullable=False)
|
|
93
|
+
signature = Column(Text, default="")
|
|
94
|
+
docstring = Column(Text, nullable=True)
|
|
95
|
+
body = Column(Text, default="")
|
|
96
|
+
body_hash = Column(String(64), default="")
|
|
97
|
+
language = Column(String(50), nullable=False)
|
|
98
|
+
complexity = Column(Integer, default=1)
|
|
99
|
+
# Phase 4.1 (roadmap §37 item 4.1; §29: "Symbol... needs... base-class
|
|
100
|
+
# field"). JSON-serialized `list[str]` of raw, unresolved base-class
|
|
101
|
+
# expressions (e.g. '["Base", "pkg.Mixin"]') -- the same "Text column
|
|
102
|
+
# holding a JSON-serialized list, parsed in application code"
|
|
103
|
+
# convention already established by `EmbeddingModel.embedding_json`,
|
|
104
|
+
# not a new serialization approach. `NULL` for a never-(re)indexed-
|
|
105
|
+
# since-this-migration row (see that migration's own docstring) and
|
|
106
|
+
# for every non-CLASS symbol; application code treats `NULL` the same
|
|
107
|
+
# as an empty list, never as "unknown."
|
|
108
|
+
base_classes = Column(Text, nullable=True)
|
|
109
|
+
|
|
110
|
+
file = relationship("FileModel", back_populates="symbols")
|
|
111
|
+
embeddings = relationship("EmbeddingModel", back_populates="symbol", cascade="all, delete-orphan")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class CallEdge(Base):
|
|
115
|
+
"""
|
|
116
|
+
Phase 3.4 (roadmap §37 item 3.4: "CallEdge.confidence scoring"; §11:
|
|
117
|
+
"every `CallEdge` should carry a `confidence` field -- `high` when
|
|
118
|
+
resolved through an explicit import mapping to a known file+symbol,
|
|
119
|
+
`low` when resolved only by bare-name match against same-file
|
|
120
|
+
symbols, `unresolved` when the callee couldn't be matched to any
|
|
121
|
+
known symbol at all"). Renamed from `DependencyModel` (table
|
|
122
|
+
`dependencies` -> `call_edges`) per §29's Data Model target entity
|
|
123
|
+
name, migration `f6e481353553`.
|
|
124
|
+
|
|
125
|
+
`confidence` is nullable: rows persisted before this migration have
|
|
126
|
+
no historical resolution-path information to backfill honestly
|
|
127
|
+
(CLAUDE.md: no fabricated metrics) and show `NULL` until their
|
|
128
|
+
repository's next re-index recomputes and persists a real value --
|
|
129
|
+
see that migration's own docstring.
|
|
130
|
+
|
|
131
|
+
RESOLVED by Phase 3.5 ("local BFS-based incremental graph
|
|
132
|
+
updates"): rows are no longer blindly appended every run. A full
|
|
133
|
+
run replaces every row for the repository; an incremental run
|
|
134
|
+
deletes and recomputes only the rows whose source symbol changed
|
|
135
|
+
this run, plus (the "local BFS one hop out") any OTHER symbol's
|
|
136
|
+
row whose target was a symbol that changed, was renamed, or
|
|
137
|
+
disappeared -- see `IndexingEngine._persist_call_graph_edges`'s own
|
|
138
|
+
docstring for the full algorithm. Before Phase 3.5 (and under the
|
|
139
|
+
`DependencyModel` name before that), rows simply accumulated
|
|
140
|
+
forever every run -- a long-disclosed Phase 0.4 limitation, now
|
|
141
|
+
closed.
|
|
142
|
+
|
|
143
|
+
`repo_id` (post-Phase-3.4 isolation fix, migration `2cd38eb12a7e`):
|
|
144
|
+
every row written by `IndexingEngine.
|
|
145
|
+
_persist_to_db` from here on carries the repository it belongs to.
|
|
146
|
+
Before this fix, `CallEdge` (and `DependencyModel` before it) had NO
|
|
147
|
+
repository association at all -- a confirmed, real cross-repository
|
|
148
|
+
data leak: `GET /v1/symbols/{id}` queried this table by symbol name
|
|
149
|
+
alone, so a caller authorized for one repository could receive
|
|
150
|
+
another, unrelated repository's call-graph edges whenever symbol
|
|
151
|
+
names happened to collide. Nullable because some historical rows
|
|
152
|
+
(persisted before this fix existed) cannot be attributed to a
|
|
153
|
+
repository with full confidence -- see that migration's own
|
|
154
|
+
docstring for the exact backfill rule (unambiguous match only, never
|
|
155
|
+
a guess). A `NULL`-`repo_id` row is invisible to every
|
|
156
|
+
repository-scoped query by construction (a real, honestly-disclosed
|
|
157
|
+
completeness gap for that specific historical edge, not a security
|
|
158
|
+
problem) until its repository is next re-indexed, which persists a
|
|
159
|
+
fresh, correctly-scoped replacement.
|
|
160
|
+
|
|
161
|
+
`relationship_type` (Phase 4.1): `"calls"` (the only kind this table
|
|
162
|
+
has ever held, and the default/backfill value for every row that
|
|
163
|
+
predates this column -- unambiguous, not a guess, since nothing else
|
|
164
|
+
was ever written here) or `"inherits"` (a resolved base-class
|
|
165
|
+
relationship, roadmap §37 item 4.1). Both kinds share this one
|
|
166
|
+
table -- same `repo_id` scoping, same `confidence` vocabulary, same
|
|
167
|
+
Phase 3.5 incremental-update machinery -- but are kept in SEPARATE
|
|
168
|
+
in-memory adjacency structures by `CallGraphBuilder`
|
|
169
|
+
(`self.adjacency`/`self.reverse_adjacency` for `"calls"` only; a
|
|
170
|
+
distinct pair for `"inherits"`), so `get_callers`/`get_callees`
|
|
171
|
+
continue to traverse call relationships only, exactly as before this
|
|
172
|
+
column existed -- see `CallGraphBuilder._build_graph`'s own
|
|
173
|
+
docstring.
|
|
174
|
+
"""
|
|
175
|
+
__tablename__ = "call_edges"
|
|
176
|
+
|
|
177
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
178
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=True, index=True)
|
|
179
|
+
source_symbol_name = Column(String(255), nullable=False, index=True)
|
|
180
|
+
source_file = Column(Text, nullable=False)
|
|
181
|
+
target_symbol_name = Column(String(255), nullable=False, index=True)
|
|
182
|
+
target_file = Column(Text, nullable=True)
|
|
183
|
+
call_count = Column(Integer, default=1)
|
|
184
|
+
relationship_type = Column(String(20), nullable=False, default="calls", server_default="calls")
|
|
185
|
+
confidence = Column(String(20), nullable=True)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class ModuleEdge(Base):
|
|
189
|
+
"""
|
|
190
|
+
The persisted file-level import graph of one repository, so a restarted
|
|
191
|
+
process can rebuild module structure without re-parsing -- the graph
|
|
192
|
+
used to exist only in `IndexingEngine.parse_results` (in memory), so
|
|
193
|
+
module structure was empty after every restart
|
|
194
|
+
(CODEPULSE_FUTURE_CHANGES.md item 15). Migration `7c3e5a91d2f4`.
|
|
195
|
+
|
|
196
|
+
One row per RESOLVED edge (`source_file` imports `target_file`, both
|
|
197
|
+
repository-local; an external or stdlib import never becomes an edge,
|
|
198
|
+
per `ModuleGraphBuilder`'s own rules). A file that has no resolved
|
|
199
|
+
imports gets one row with `target_file` NULL: the node set is stored
|
|
200
|
+
explicitly, so (a) isolated files are kept, (b) the restored graph has
|
|
201
|
+
exactly the files that were parsed (not whatever `files` rows exist,
|
|
202
|
+
which can include stale rows), and (c) "no rows for a repository"
|
|
203
|
+
reliably means "not indexed since this table existed" -- the repository
|
|
204
|
+
stays cold instead of showing a misleading all-singletons graph.
|
|
205
|
+
|
|
206
|
+
Rows are replaced wholesale for the repository on every index run (full
|
|
207
|
+
or incremental) from the complete merged parse set, so they are always
|
|
208
|
+
a coherent snapshot and re-indexing never duplicates them. `repo_id` is
|
|
209
|
+
NOT NULL and cascades on repository deletion.
|
|
210
|
+
"""
|
|
211
|
+
__tablename__ = "module_edges"
|
|
212
|
+
__table_args__ = (
|
|
213
|
+
UniqueConstraint("repo_id", "source_file", "target_file", name="uq_module_edges_repo_source_target"),
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
217
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
|
|
218
|
+
source_file = Column(Text, nullable=False)
|
|
219
|
+
target_file = Column(Text, nullable=True)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class EmbeddingModel(Base):
|
|
223
|
+
__tablename__ = "code_embeddings"
|
|
224
|
+
__table_args__ = (
|
|
225
|
+
# Natural key for embedding identity (Phase 0.5), per the roadmap's
|
|
226
|
+
# Phase 0 idempotency rule (§8): "(symbol_id, chunk_type,
|
|
227
|
+
# chunk_index)". Enforced at the DB level so a duplicate Embedding
|
|
228
|
+
# row can never be created for the same chunk of the same symbol.
|
|
229
|
+
UniqueConstraint(
|
|
230
|
+
"symbol_id", "chunk_type", "chunk_index", name="uq_code_embeddings_symbol_chunk"
|
|
231
|
+
),
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
235
|
+
symbol_id = Column(Integer, ForeignKey("symbols.id", ondelete="CASCADE"), nullable=False)
|
|
236
|
+
chunk_type = Column(String(50), default="signature_doc") # 'signature_doc', 'body_chunk', 'full'
|
|
237
|
+
# Position of this chunk among a symbol's chunks (see
|
|
238
|
+
# codepulse.embeddings.chunker.CodeChunk.chunk_index); part of the
|
|
239
|
+
# natural key together with (symbol_id, chunk_type).
|
|
240
|
+
chunk_index = Column(Integer, nullable=False, default=0)
|
|
241
|
+
embedding_json = Column(Text, nullable=False) # JSON serialized array of floats for portability
|
|
242
|
+
chunk_content = Column(Text, default="")
|
|
243
|
+
# SHA-256 hex digest of `chunk_content` (see codepulse.models.compute_content_hash),
|
|
244
|
+
# used for change detection: reuse the row if unchanged, update in place
|
|
245
|
+
# if the chunk's source content changed. Nullable because historical
|
|
246
|
+
# rows created before Phase 0.5 have no recorded hash.
|
|
247
|
+
content_hash = Column(String(64), nullable=True)
|
|
248
|
+
# Identity of the model that produced this vector, and its dimension --
|
|
249
|
+
# recorded so hydration can detect a model/dimension change and skip
|
|
250
|
+
# loading an incompatible vector rather than silently corrupting the
|
|
251
|
+
# in-memory vector index (roadmap §9: "embedding dimension/model must be
|
|
252
|
+
# recorded"). Nullable for the same reason as content_hash.
|
|
253
|
+
embedding_model = Column(String(150), nullable=True)
|
|
254
|
+
embedding_dim = Column(Integer, nullable=True)
|
|
255
|
+
|
|
256
|
+
symbol = relationship("SymbolModel", back_populates="embeddings")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class APIKeyModel(Base):
|
|
260
|
+
"""
|
|
261
|
+
Phase 1.1 (roadmap §15, §37 item 1.1): API key model + hashing +
|
|
262
|
+
creation flow.
|
|
263
|
+
|
|
264
|
+
The raw key is NEVER stored -- only its SHA-256 hash (`key_hash`, see
|
|
265
|
+
codepulse.auth.hash_api_key) and a short, non-secret prefix
|
|
266
|
+
(`key_prefix`) kept purely for display/identification in a future
|
|
267
|
+
key-management UI/CLI. This is Phase 1.1 scope only: the model,
|
|
268
|
+
hashing, and a creation (+ verification-primitive) flow. Enforcing
|
|
269
|
+
this at the API boundary (a FastAPI dependency required on every
|
|
270
|
+
route) is Phase 1.2, deliberately not implemented here. Role-based
|
|
271
|
+
permissions (viewer/indexer/admin, roadmap §15) were Phase 7 (B25)
|
|
272
|
+
scope -- see `APIKeyRepositoryModel.role`'s own docstring; the role
|
|
273
|
+
lives on the per-repository grant row, not here on `APIKeyModel`
|
|
274
|
+
itself, per §15's own "enforced at the repository-permission level,
|
|
275
|
+
not globally."
|
|
276
|
+
"""
|
|
277
|
+
__tablename__ = "api_keys"
|
|
278
|
+
__table_args__ = (
|
|
279
|
+
UniqueConstraint("key_hash", name="uq_api_keys_key_hash"),
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
283
|
+
name = Column(String(255), nullable=False)
|
|
284
|
+
# SHA-256 hex digest of the raw key (codepulse.auth.hash_api_key). A
|
|
285
|
+
# fast cryptographic hash (not a slow/salted password hash) is
|
|
286
|
+
# appropriate here because the input is already a high-entropy,
|
|
287
|
+
# randomly generated secret (codepulse.auth.generate_raw_api_key), not
|
|
288
|
+
# a user-chosen low-entropy password -- and a fast hash keeps
|
|
289
|
+
# lookup-by-hash a simple indexed equality query for Phase 1.2's
|
|
290
|
+
# authorization dependency.
|
|
291
|
+
key_hash = Column(String(64), nullable=False)
|
|
292
|
+
# Short, non-secret prefix of the raw key (e.g. "cpk_a1b2c3d4"), kept
|
|
293
|
+
# only so a key can be identified/distinguished in a future listing
|
|
294
|
+
# UI/CLI without ever re-displaying the secret itself.
|
|
295
|
+
key_prefix = Column(String(32), nullable=False)
|
|
296
|
+
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
|
297
|
+
# Nullable revocation timestamp: schema support only in Phase 1.1 (so
|
|
298
|
+
# a later revocation milestone doesn't need its own migration) -- no
|
|
299
|
+
# revoke() flow is implemented yet; codepulse.auth.verify_api_key does
|
|
300
|
+
# respect this field defensively (a revoked key never verifies).
|
|
301
|
+
revoked_at = Column(DateTime, nullable=True)
|
|
302
|
+
|
|
303
|
+
repositories = relationship(
|
|
304
|
+
"APIKeyRepositoryModel", back_populates="api_key", cascade="all, delete-orphan"
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
class APIKeyRepositoryModel(Base):
|
|
309
|
+
"""
|
|
310
|
+
Scopes an APIKey to a specific repository (roadmap §15: "API keys
|
|
311
|
+
(hashed at rest, scoped to one or more repositories)"). One row per
|
|
312
|
+
(api_key, repository) pair a key is permitted to access. This table
|
|
313
|
+
only records the grant -- Phase 1.1 does not read or enforce it
|
|
314
|
+
anywhere; that's Phase 1.2's central authorization dependency.
|
|
315
|
+
|
|
316
|
+
Phase 7.2 (roadmap §37 item 7.2 "RBAC roles + repo permission
|
|
317
|
+
model"; §15's own literal minimal-viable model: *"roles `viewer`
|
|
318
|
+
(search/read), `indexer` (viewer + trigger indexing), `admin` (all +
|
|
319
|
+
manage keys/repos). Enforced at the repository-permission level, not
|
|
320
|
+
globally."*): `role` lives HERE, on the per-(key, repository) grant
|
|
321
|
+
row, not on `APIKeyModel` itself -- exactly matching "not globally."
|
|
322
|
+
A single key can legitimately be `viewer` on one repository and
|
|
323
|
+
`admin` on another.
|
|
324
|
+
|
|
325
|
+
`NOT NULL DEFAULT 'admin'`: every grant created before this
|
|
326
|
+
milestone had, in practice, UNRESTRICTED access to every endpoint
|
|
327
|
+
that checked repository membership alone (no role concept existed
|
|
328
|
+
to restrict it) -- backfilling to `'admin'` reflects each existing
|
|
329
|
+
grant's actual historical capability honestly, not a guess (unlike
|
|
330
|
+
an ambiguous backfill elsewhere in this project's history, this one
|
|
331
|
+
has a single, unambiguous correct answer). `codepulse.auth.
|
|
332
|
+
create_api_key`'s own default for NEWLY created grants going
|
|
333
|
+
forward is intentionally the safer `'viewer'` (least privilege),
|
|
334
|
+
matching this table's forward-looking correctness rather than its
|
|
335
|
+
backfilled historical value.
|
|
336
|
+
"""
|
|
337
|
+
__tablename__ = "api_key_repositories"
|
|
338
|
+
__table_args__ = (
|
|
339
|
+
UniqueConstraint("api_key_id", "repo_id", name="uq_api_key_repositories_key_repo"),
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
343
|
+
api_key_id = Column(Integer, ForeignKey("api_keys.id", ondelete="CASCADE"), nullable=False)
|
|
344
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False)
|
|
345
|
+
role = Column(String(20), nullable=False, default="admin", server_default="admin")
|
|
346
|
+
|
|
347
|
+
api_key = relationship("APIKeyModel", back_populates="repositories")
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class IndexJobModel(Base):
|
|
351
|
+
"""
|
|
352
|
+
Phase 1.3 (roadmap §37 item 1.3; §8 "Concurrency/failure handling";
|
|
353
|
+
§29 Data Model; backlog B9). Tracks one indexing run for a repository
|
|
354
|
+
through the state machine roadmap §8 describes literally: `queued ->
|
|
355
|
+
running -> succeeded|failed|cancelled`. Replaces the previous fake
|
|
356
|
+
`job_id` (`f"job_{int(time.time())}"`, no backing state at all) that
|
|
357
|
+
`/v1/repositories/index` used to return -- B9's DoD: "Returned job_id
|
|
358
|
+
is pollable to real completion" (see `GET /v1/jobs/{job_id}`).
|
|
359
|
+
|
|
360
|
+
Unlike every other model in this file, this table is NOT an
|
|
361
|
+
upsert-by-natural-key entity (no `UniqueConstraint` here) -- a job is a
|
|
362
|
+
record of one indexing *run* (an event), not an idempotent identity to
|
|
363
|
+
be reused across runs; every indexing attempt for a repository gets
|
|
364
|
+
its own row, by design, forming a run history. This does not weaken
|
|
365
|
+
the project's "idempotent indexing" invariant, which governs the
|
|
366
|
+
File/Symbol/Embedding rows an indexing run itself writes -- unaffected
|
|
367
|
+
by this table.
|
|
368
|
+
|
|
369
|
+
`cancelled` is a valid, permanent schema value (so a later
|
|
370
|
+
cancellation milestone doesn't need its own migration) but is never
|
|
371
|
+
actually written by this milestone -- no cancel flow/endpoint exists
|
|
372
|
+
yet, matching how Phase 1.1 left `APIKeyModel.revoked_at` schema-only
|
|
373
|
+
until a revoke() flow existed.
|
|
374
|
+
|
|
375
|
+
Row lifecycle logic (create / transition / query) lives in
|
|
376
|
+
`codepulse.jobs`, not here and not in `IndexingEngine` -- this table
|
|
377
|
+
tracks *orchestration* state for one indexing run, the same
|
|
378
|
+
boundary-level category as `APIKeyModel`/`AuditEvent` in the roadmap's
|
|
379
|
+
own §29 Data Model table, not a core intelligence entity.
|
|
380
|
+
"""
|
|
381
|
+
__tablename__ = "index_jobs"
|
|
382
|
+
|
|
383
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
384
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
|
|
385
|
+
status = Column(String(20), nullable=False, default="queued")
|
|
386
|
+
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
|
387
|
+
started_at = Column(DateTime, nullable=True)
|
|
388
|
+
finished_at = Column(DateTime, nullable=True)
|
|
389
|
+
error_message = Column(Text, nullable=True)
|
|
390
|
+
# JSON-serialized copy of IndexingEngine.index_directory()'s real
|
|
391
|
+
# return dict (repository_name, total_files, total_symbols, ... -- see
|
|
392
|
+
# engine.py) once the job succeeds. Never fabricated/estimated --
|
|
393
|
+
# populated only from the actual measured result, per this project's
|
|
394
|
+
# "no fabricated metrics" invariant.
|
|
395
|
+
result_summary = Column(Text, nullable=True)
|
|
396
|
+
|
|
397
|
+
repository = relationship("RepositoryModel")
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class RepositorySnapshotModel(Base):
|
|
401
|
+
"""
|
|
402
|
+
Phase 3.1 (roadmap §37 item 3.1: "Git-diff change detection
|
|
403
|
+
(added/modified/deleted)"; §9 Incremental Indexing Design; §29 Data
|
|
404
|
+
Model, which explicitly names `RepositorySnapshot` as a required-now
|
|
405
|
+
entity: "Required for incremental indexing (§9) and crash-safe reads
|
|
406
|
+
(§17)"). Records, for one successful indexing run, the git commit
|
|
407
|
+
the repository's content was at (`commit_sha`, `None` for a
|
|
408
|
+
non-git/local-path repository -- git-diff-based change detection is
|
|
409
|
+
only possible for a real git repository) and the embedding model/
|
|
410
|
+
dimension used -- both are exactly the two pieces of state §9's own
|
|
411
|
+
"when a full re-index is still required" list needs to check on the
|
|
412
|
+
*next* run: "if no prior successful snapshot exists, [or] if the
|
|
413
|
+
embedding model version changed".
|
|
414
|
+
|
|
415
|
+
Like `IndexJobModel`, this is a run-history table (no natural-key
|
|
416
|
+
`UniqueConstraint`) -- one row per successful `index_directory()`
|
|
417
|
+
call, not an upsert-by-identity entity. `IndexingEngine._plan_
|
|
418
|
+
incremental_run` reads the single most recent row for a `repo_id` to
|
|
419
|
+
decide whether an incremental (git-diff-driven) run is possible; a
|
|
420
|
+
full re-index still creates its own new row here exactly the same
|
|
421
|
+
way, so "no prior successful snapshot" naturally self-heals after
|
|
422
|
+
the very first successful run.
|
|
423
|
+
|
|
424
|
+
This milestone does NOT implement §17's full crash-safe,
|
|
425
|
+
status-gated "always serve the latest ready snapshot" read semantics
|
|
426
|
+
-- that is a broader reliability feature §29 attributes to this same
|
|
427
|
+
entity for a *later* purpose, not something Phase 3.1's own literal
|
|
428
|
+
"git-diff change detection" scope requires building now. No `status`
|
|
429
|
+
column exists here for that reason; disclosed, not silently
|
|
430
|
+
expanded into.
|
|
431
|
+
"""
|
|
432
|
+
__tablename__ = "repository_snapshots"
|
|
433
|
+
|
|
434
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
435
|
+
repo_id = Column(Integer, ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False, index=True)
|
|
436
|
+
commit_sha = Column(String(64), nullable=True)
|
|
437
|
+
embedding_model = Column(String(255), nullable=True)
|
|
438
|
+
embedding_dim = Column(Integer, nullable=True)
|
|
439
|
+
indexed_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
|
440
|
+
|
|
441
|
+
repository = relationship("RepositoryModel")
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class StorageManager:
|
|
445
|
+
"""Manages database connection, tables, and session lifecycle."""
|
|
446
|
+
|
|
447
|
+
def __init__(self, db_url: str | None = None) -> None:
|
|
448
|
+
if db_url is None:
|
|
449
|
+
from codepulse.config import get_settings
|
|
450
|
+
db_url = get_settings().db_url
|
|
451
|
+
self.db_url = db_url
|
|
452
|
+
# The per-user data directory is created on demand; any other missing directory is still an error.
|
|
453
|
+
from codepulse.data_location import ensure_database_directory
|
|
454
|
+
|
|
455
|
+
ensure_database_directory(db_url)
|
|
456
|
+
self.engine = create_engine(db_url, echo=False)
|
|
457
|
+
|
|
458
|
+
# Backend-hardening addition (not a roadmap sprint item): SQLite
|
|
459
|
+
# disables foreign-key enforcement by default, per-connection --
|
|
460
|
+
# every ON DELETE CASCADE declared on a ForeignKey() above (e.g.
|
|
461
|
+
# CallEdge.repo_id, APIKeyRepositoryModel.api_key_id/repo_id,
|
|
462
|
+
# IndexJobModel.repo_id, RepositorySnapshotModel.repo_id) was
|
|
463
|
+
# previously inert, a real, disclosed gap (see CODEPULSE_PROGRESS.md's
|
|
464
|
+
# "Known technical debt"). Scoped to THIS engine instance only
|
|
465
|
+
# (not the global sqlalchemy.engine.Engine class) so it applies to
|
|
466
|
+
# every connection the application makes through a StorageManager,
|
|
467
|
+
# while deliberately NOT touching Alembic's own, entirely separate
|
|
468
|
+
# `engine_from_config(...)` connection (alembic/env.py) -- migration
|
|
469
|
+
# execution is unaffected by this change. Guarded to SQLite only,
|
|
470
|
+
# since PRAGMA is SQLite-specific syntax and Postgres (Phase 7.1,
|
|
471
|
+
# not implemented) enforces foreign keys by default already, no
|
|
472
|
+
# pragma needed.
|
|
473
|
+
@event.listens_for(self.engine, "connect")
|
|
474
|
+
def _enable_sqlite_foreign_keys(dbapi_connection, connection_record) -> None:
|
|
475
|
+
if self.engine.dialect.name != "sqlite":
|
|
476
|
+
return
|
|
477
|
+
cursor = dbapi_connection.cursor()
|
|
478
|
+
cursor.execute("PRAGMA foreign_keys=ON")
|
|
479
|
+
cursor.close()
|
|
480
|
+
|
|
481
|
+
self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self.engine)
|
|
482
|
+
self.create_tables()
|
|
483
|
+
|
|
484
|
+
def create_tables(self) -> None:
|
|
485
|
+
"""
|
|
486
|
+
Creates every table declared on `Base.metadata` that doesn't yet
|
|
487
|
+
exist -- but ONLY on a database Alembic has never touched (no
|
|
488
|
+
`alembic_version` table present). Backend-hardening fix, closing
|
|
489
|
+
the actual root cause of three separate, real, documented
|
|
490
|
+
incidents (see CODEPULSE_PROGRESS.md's "Known technical debt"):
|
|
491
|
+
an ad-hoc `StorageManager()` construction against the REAL,
|
|
492
|
+
already-migrated `codepulse.db` (e.g. importing
|
|
493
|
+
`codepulse.api.app` without an isolated `CODEPULSE_DB_URL`)
|
|
494
|
+
could silently `create_all()` a table a pending Alembic
|
|
495
|
+
migration hadn't created yet, racing ahead of Alembic and
|
|
496
|
+
leaving `alembic_version` stale -- requiring manual
|
|
497
|
+
remediation each time it happened.
|
|
498
|
+
|
|
499
|
+
A fresh, disposable database (every test fixture in this
|
|
500
|
+
project's own suite, e.g. `tests/conftest.py`'s `isolated_app`)
|
|
501
|
+
has no `alembic_version` table yet, so this remains a true
|
|
502
|
+
zero-setup `create_all()` for that case, unchanged from before
|
|
503
|
+
this fix -- `tests/test_migrations.py::
|
|
504
|
+
test_existing_database_stamping_scenario_2` exercises exactly
|
|
505
|
+
this "no alembic_version yet" precondition directly. The real
|
|
506
|
+
`codepulse.db` has had an `alembic_version` table since its very
|
|
507
|
+
first migration; from that point on, this method is a safe
|
|
508
|
+
no-op for it, and schema changes must go through Alembic only.
|
|
509
|
+
"""
|
|
510
|
+
inspector = inspect(self.engine)
|
|
511
|
+
if inspector.has_table("alembic_version"):
|
|
512
|
+
return
|
|
513
|
+
brand_new = not inspector.get_table_names()
|
|
514
|
+
Base.metadata.create_all(bind=self.engine)
|
|
515
|
+
if brand_new:
|
|
516
|
+
# A database created from scratch here IS the current schema, so record that. Until
|
|
517
|
+
# 2026-10-14 such databases were never tracked by Alembic and could not be upgraded
|
|
518
|
+
# later (FUTURE_CHANGES items 8/17). A database that already had tables but no
|
|
519
|
+
# alembic_version (created by older code) is deliberately NOT stamped here: whether
|
|
520
|
+
# its schema matches is what `codepulse migrate` checks before it stamps.
|
|
521
|
+
from codepulse.storage.migrations import stamp_new_database
|
|
522
|
+
|
|
523
|
+
stamp_new_database(self.engine)
|
|
524
|
+
|
|
525
|
+
def get_session(self) -> Session:
|
|
526
|
+
return self.SessionLocal()
|