cinchdb 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cinchdb/cli/commands/column.py +3 -4
- cinchdb/cli/commands/database.py +58 -60
- cinchdb/cli/commands/table.py +3 -3
- cinchdb/cli/main.py +1 -7
- cinchdb/cli/utils.py +23 -0
- cinchdb/core/database.py +138 -11
- cinchdb/core/initializer.py +188 -10
- cinchdb/core/path_utils.py +44 -22
- cinchdb/infrastructure/metadata_connection_pool.py +145 -0
- cinchdb/infrastructure/metadata_db.py +376 -0
- cinchdb/managers/branch.py +119 -23
- cinchdb/managers/change_applier.py +30 -13
- cinchdb/managers/column.py +4 -10
- cinchdb/managers/query.py +40 -4
- cinchdb/managers/table.py +8 -6
- cinchdb/managers/tenant.py +718 -96
- cinchdb/models/table.py +0 -4
- cinchdb/models/tenant.py +4 -2
- {cinchdb-0.1.9.dist-info → cinchdb-0.1.11.dist-info}/METADATA +5 -36
- {cinchdb-0.1.9.dist-info → cinchdb-0.1.11.dist-info}/RECORD +23 -21
- {cinchdb-0.1.9.dist-info → cinchdb-0.1.11.dist-info}/WHEEL +0 -0
- {cinchdb-0.1.9.dist-info → cinchdb-0.1.11.dist-info}/entry_points.txt +0 -0
- {cinchdb-0.1.9.dist-info → cinchdb-0.1.11.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,376 @@
|
|
1
|
+
"""SQLite-based metadata storage for lazy resource tracking."""
|
2
|
+
|
3
|
+
import sqlite3
|
4
|
+
import uuid
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import Optional, List, Dict, Any
|
7
|
+
from datetime import datetime
|
8
|
+
import json
|
9
|
+
|
10
|
+
|
11
|
+
class MetadataDB:
|
12
|
+
"""Manages SQLite database for project metadata."""
|
13
|
+
|
14
|
+
def __init__(self, project_path: Path):
|
15
|
+
"""Initialize metadata database for a project."""
|
16
|
+
self.db_path = project_path / ".cinchdb" / "metadata.db"
|
17
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
18
|
+
self.conn: Optional[sqlite3.Connection] = None
|
19
|
+
self._connect()
|
20
|
+
self._create_tables()
|
21
|
+
|
22
|
+
def _connect(self):
|
23
|
+
"""Connect to the SQLite database."""
|
24
|
+
# Check if this is a new database
|
25
|
+
is_new_db = not self.db_path.exists()
|
26
|
+
|
27
|
+
self.conn = sqlite3.connect(
|
28
|
+
str(self.db_path),
|
29
|
+
check_same_thread=False, # Allow multi-threaded access
|
30
|
+
timeout=30.0
|
31
|
+
)
|
32
|
+
self.conn.row_factory = sqlite3.Row
|
33
|
+
|
34
|
+
# For new databases, set small page size before creating any tables
|
35
|
+
if is_new_db:
|
36
|
+
self.conn.execute("PRAGMA page_size = 1024") # 1KB pages for metadata DB
|
37
|
+
|
38
|
+
self.conn.execute("PRAGMA foreign_keys = ON")
|
39
|
+
self.conn.execute("PRAGMA journal_mode = WAL") # Better concurrency
|
40
|
+
|
41
|
+
def _create_tables(self):
|
42
|
+
"""Create the metadata tables if they don't exist."""
|
43
|
+
with self.conn:
|
44
|
+
# Databases table
|
45
|
+
self.conn.execute("""
|
46
|
+
CREATE TABLE IF NOT EXISTS databases (
|
47
|
+
id TEXT PRIMARY KEY,
|
48
|
+
name TEXT NOT NULL UNIQUE,
|
49
|
+
description TEXT,
|
50
|
+
materialized BOOLEAN DEFAULT FALSE,
|
51
|
+
metadata JSON,
|
52
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
53
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
54
|
+
)
|
55
|
+
""")
|
56
|
+
|
57
|
+
# Branches table
|
58
|
+
self.conn.execute("""
|
59
|
+
CREATE TABLE IF NOT EXISTS branches (
|
60
|
+
id TEXT PRIMARY KEY,
|
61
|
+
database_id TEXT NOT NULL,
|
62
|
+
name TEXT NOT NULL,
|
63
|
+
parent_branch TEXT,
|
64
|
+
schema_version TEXT,
|
65
|
+
materialized BOOLEAN DEFAULT FALSE,
|
66
|
+
metadata JSON,
|
67
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
68
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
69
|
+
FOREIGN KEY (database_id) REFERENCES databases(id) ON DELETE CASCADE,
|
70
|
+
UNIQUE(database_id, name)
|
71
|
+
)
|
72
|
+
""")
|
73
|
+
|
74
|
+
# Tenants table
|
75
|
+
self.conn.execute("""
|
76
|
+
CREATE TABLE IF NOT EXISTS tenants (
|
77
|
+
id TEXT PRIMARY KEY,
|
78
|
+
branch_id TEXT NOT NULL,
|
79
|
+
name TEXT NOT NULL,
|
80
|
+
shard TEXT,
|
81
|
+
materialized BOOLEAN DEFAULT FALSE,
|
82
|
+
metadata JSON,
|
83
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
84
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
85
|
+
FOREIGN KEY (branch_id) REFERENCES branches(id) ON DELETE CASCADE,
|
86
|
+
UNIQUE(branch_id, name)
|
87
|
+
)
|
88
|
+
""")
|
89
|
+
|
90
|
+
# Create indexes for common queries
|
91
|
+
self.conn.execute("""
|
92
|
+
CREATE INDEX IF NOT EXISTS idx_branches_database
|
93
|
+
ON branches(database_id)
|
94
|
+
""")
|
95
|
+
|
96
|
+
self.conn.execute("""
|
97
|
+
CREATE INDEX IF NOT EXISTS idx_tenants_branch
|
98
|
+
ON tenants(branch_id)
|
99
|
+
""")
|
100
|
+
|
101
|
+
self.conn.execute("""
|
102
|
+
CREATE INDEX IF NOT EXISTS idx_databases_materialized
|
103
|
+
ON databases(materialized)
|
104
|
+
""")
|
105
|
+
|
106
|
+
self.conn.execute("""
|
107
|
+
CREATE INDEX IF NOT EXISTS idx_branches_materialized
|
108
|
+
ON branches(materialized)
|
109
|
+
""")
|
110
|
+
|
111
|
+
self.conn.execute("""
|
112
|
+
CREATE INDEX IF NOT EXISTS idx_tenants_materialized
|
113
|
+
ON tenants(materialized)
|
114
|
+
""")
|
115
|
+
|
116
|
+
self.conn.execute("""
|
117
|
+
CREATE INDEX IF NOT EXISTS idx_tenants_shard
|
118
|
+
ON tenants(shard)
|
119
|
+
""")
|
120
|
+
|
121
|
+
# Database operations
|
122
|
+
def create_database(self, database_id: str, name: str,
|
123
|
+
description: Optional[str] = None,
|
124
|
+
metadata: Optional[Dict[str, Any]] = None) -> None:
|
125
|
+
"""Create a lazy database entry."""
|
126
|
+
with self.conn:
|
127
|
+
self.conn.execute("""
|
128
|
+
INSERT INTO databases (id, name, description, metadata)
|
129
|
+
VALUES (?, ?, ?, ?)
|
130
|
+
""", (database_id, name, description,
|
131
|
+
json.dumps(metadata) if metadata else None))
|
132
|
+
|
133
|
+
def get_database(self, name: str) -> Optional[Dict[str, Any]]:
|
134
|
+
"""Get database by name."""
|
135
|
+
cursor = self.conn.execute("""
|
136
|
+
SELECT * FROM databases WHERE name = ?
|
137
|
+
""", (name,))
|
138
|
+
row = cursor.fetchone()
|
139
|
+
return dict(row) if row else None
|
140
|
+
|
141
|
+
def list_databases(self, materialized_only: bool = False) -> List[Dict[str, Any]]:
|
142
|
+
"""List all databases."""
|
143
|
+
query = "SELECT * FROM databases"
|
144
|
+
if materialized_only:
|
145
|
+
query += " WHERE materialized = TRUE"
|
146
|
+
cursor = self.conn.execute(query)
|
147
|
+
return [dict(row) for row in cursor.fetchall()]
|
148
|
+
|
149
|
+
def mark_database_materialized(self, database_id: str) -> None:
|
150
|
+
"""Mark a database as materialized."""
|
151
|
+
with self.conn:
|
152
|
+
self.conn.execute("""
|
153
|
+
UPDATE databases
|
154
|
+
SET materialized = TRUE, updated_at = CURRENT_TIMESTAMP
|
155
|
+
WHERE id = ?
|
156
|
+
""", (database_id,))
|
157
|
+
|
158
|
+
# Branch operations
|
159
|
+
def create_branch(self, branch_id: str, database_id: str, name: str,
|
160
|
+
parent_branch: Optional[str] = None,
|
161
|
+
schema_version: Optional[str] = None,
|
162
|
+
metadata: Optional[Dict[str, Any]] = None) -> None:
|
163
|
+
"""Create a lazy branch entry."""
|
164
|
+
with self.conn:
|
165
|
+
self.conn.execute("""
|
166
|
+
INSERT INTO branches (id, database_id, name, parent_branch,
|
167
|
+
schema_version, metadata)
|
168
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
169
|
+
""", (branch_id, database_id, name, parent_branch, schema_version,
|
170
|
+
json.dumps(metadata) if metadata else None))
|
171
|
+
|
172
|
+
def get_branch(self, database_id: str, name: str) -> Optional[Dict[str, Any]]:
|
173
|
+
"""Get branch by database and name."""
|
174
|
+
cursor = self.conn.execute("""
|
175
|
+
SELECT * FROM branches
|
176
|
+
WHERE database_id = ? AND name = ?
|
177
|
+
""", (database_id, name))
|
178
|
+
row = cursor.fetchone()
|
179
|
+
return dict(row) if row else None
|
180
|
+
|
181
|
+
def list_branches(self, database_id: str,
|
182
|
+
materialized_only: bool = False) -> List[Dict[str, Any]]:
|
183
|
+
"""List branches for a database."""
|
184
|
+
query = "SELECT * FROM branches WHERE database_id = ?"
|
185
|
+
params = [database_id]
|
186
|
+
if materialized_only:
|
187
|
+
query += " AND materialized = TRUE"
|
188
|
+
cursor = self.conn.execute(query, params)
|
189
|
+
return [dict(row) for row in cursor.fetchall()]
|
190
|
+
|
191
|
+
def mark_branch_materialized(self, branch_id: str) -> None:
|
192
|
+
"""Mark a branch as materialized."""
|
193
|
+
with self.conn:
|
194
|
+
self.conn.execute("""
|
195
|
+
UPDATE branches
|
196
|
+
SET materialized = TRUE, updated_at = CURRENT_TIMESTAMP
|
197
|
+
WHERE id = ?
|
198
|
+
""", (branch_id,))
|
199
|
+
|
200
|
+
# Tenant operations
|
201
|
+
def create_tenant(self, tenant_id: str, branch_id: str, name: str,
|
202
|
+
shard: Optional[str] = None,
|
203
|
+
metadata: Optional[Dict[str, Any]] = None) -> None:
|
204
|
+
"""Create a lazy tenant entry."""
|
205
|
+
with self.conn:
|
206
|
+
self.conn.execute("""
|
207
|
+
INSERT INTO tenants (id, branch_id, name, shard, metadata)
|
208
|
+
VALUES (?, ?, ?, ?, ?)
|
209
|
+
""", (tenant_id, branch_id, name, shard,
|
210
|
+
json.dumps(metadata) if metadata else None))
|
211
|
+
|
212
|
+
def get_tenant(self, branch_id: str, name: str) -> Optional[Dict[str, Any]]:
|
213
|
+
"""Get tenant by branch and name."""
|
214
|
+
cursor = self.conn.execute("""
|
215
|
+
SELECT * FROM tenants
|
216
|
+
WHERE branch_id = ? AND name = ?
|
217
|
+
""", (branch_id, name))
|
218
|
+
row = cursor.fetchone()
|
219
|
+
return dict(row) if row else None
|
220
|
+
|
221
|
+
def list_tenants(self, branch_id: str,
|
222
|
+
materialized_only: bool = False) -> List[Dict[str, Any]]:
|
223
|
+
"""List tenants for a branch."""
|
224
|
+
query = "SELECT * FROM tenants WHERE branch_id = ?"
|
225
|
+
params = [branch_id]
|
226
|
+
if materialized_only:
|
227
|
+
query += " AND materialized = TRUE"
|
228
|
+
cursor = self.conn.execute(query, params)
|
229
|
+
return [dict(row) for row in cursor.fetchall()]
|
230
|
+
|
231
|
+
def get_tenants_by_shard(self, branch_id: str, shard: str) -> List[Dict[str, Any]]:
|
232
|
+
"""Get all tenants in a specific shard for a branch."""
|
233
|
+
cursor = self.conn.execute("""
|
234
|
+
SELECT * FROM tenants
|
235
|
+
WHERE branch_id = ? AND shard = ?
|
236
|
+
""", (branch_id, shard))
|
237
|
+
return [dict(row) for row in cursor.fetchall()]
|
238
|
+
|
239
|
+
def mark_tenant_materialized(self, tenant_id: str) -> None:
|
240
|
+
"""Mark a tenant as materialized."""
|
241
|
+
with self.conn:
|
242
|
+
self.conn.execute("""
|
243
|
+
UPDATE tenants
|
244
|
+
SET materialized = TRUE, updated_at = CURRENT_TIMESTAMP
|
245
|
+
WHERE id = ?
|
246
|
+
""", (tenant_id,))
|
247
|
+
|
248
|
+
def delete_database(self, database_id: str) -> None:
|
249
|
+
"""Delete a database and all its branches and tenants (cascading delete)."""
|
250
|
+
with self.conn:
|
251
|
+
cursor = self.conn.execute("""
|
252
|
+
DELETE FROM databases WHERE id = ?
|
253
|
+
""", (database_id,))
|
254
|
+
if cursor.rowcount == 0:
|
255
|
+
raise ValueError(f"Database with id {database_id} not found")
|
256
|
+
|
257
|
+
def delete_database_by_name(self, name: str) -> None:
|
258
|
+
"""Delete a database by name and all its branches and tenants (cascading delete)."""
|
259
|
+
with self.conn:
|
260
|
+
cursor = self.conn.execute("""
|
261
|
+
DELETE FROM databases WHERE name = ?
|
262
|
+
""", (name,))
|
263
|
+
if cursor.rowcount == 0:
|
264
|
+
raise ValueError(f"Database '{name}' not found")
|
265
|
+
|
266
|
+
def delete_branch(self, branch_id: str) -> None:
|
267
|
+
"""Delete a branch and all its tenants (cascading delete)."""
|
268
|
+
with self.conn:
|
269
|
+
cursor = self.conn.execute("""
|
270
|
+
DELETE FROM branches WHERE id = ?
|
271
|
+
""", (branch_id,))
|
272
|
+
if cursor.rowcount == 0:
|
273
|
+
raise ValueError(f"Branch with id {branch_id} not found")
|
274
|
+
|
275
|
+
def delete_branch_by_name(self, database_id: str, branch_name: str) -> None:
|
276
|
+
"""Delete a branch by name and all its tenants (cascading delete)."""
|
277
|
+
with self.conn:
|
278
|
+
cursor = self.conn.execute("""
|
279
|
+
DELETE FROM branches WHERE database_id = ? AND name = ?
|
280
|
+
""", (database_id, branch_name))
|
281
|
+
if cursor.rowcount == 0:
|
282
|
+
raise ValueError(f"Branch '{branch_name}' not found in database")
|
283
|
+
|
284
|
+
def delete_tenant(self, tenant_id: str) -> None:
|
285
|
+
"""Delete a tenant."""
|
286
|
+
with self.conn:
|
287
|
+
cursor = self.conn.execute("""
|
288
|
+
DELETE FROM tenants WHERE id = ?
|
289
|
+
""", (tenant_id,))
|
290
|
+
if cursor.rowcount == 0:
|
291
|
+
raise ValueError(f"Tenant with id {tenant_id} not found")
|
292
|
+
|
293
|
+
def delete_tenant_by_name(self, branch_id: str, tenant_name: str) -> None:
|
294
|
+
"""Delete a tenant by name."""
|
295
|
+
with self.conn:
|
296
|
+
cursor = self.conn.execute("""
|
297
|
+
DELETE FROM tenants WHERE branch_id = ? AND name = ?
|
298
|
+
""", (branch_id, tenant_name))
|
299
|
+
if cursor.rowcount == 0:
|
300
|
+
raise ValueError(f"Tenant '{tenant_name}' not found in branch")
|
301
|
+
|
302
|
+
def tenant_exists(self, database_name: str, branch_name: str,
|
303
|
+
tenant_name: str) -> bool:
|
304
|
+
"""Check if a tenant exists (lazy or materialized)."""
|
305
|
+
cursor = self.conn.execute("""
|
306
|
+
SELECT 1 FROM tenants t
|
307
|
+
JOIN branches b ON t.branch_id = b.id
|
308
|
+
JOIN databases d ON b.database_id = d.id
|
309
|
+
WHERE d.name = ? AND b.name = ? AND t.name = ?
|
310
|
+
LIMIT 1
|
311
|
+
""", (database_name, branch_name, tenant_name))
|
312
|
+
return cursor.fetchone() is not None
|
313
|
+
|
314
|
+
def get_full_tenant_info(self, database_name: str, branch_name: str,
|
315
|
+
tenant_name: str) -> Optional[Dict[str, Any]]:
|
316
|
+
"""Get full tenant information including database and branch details."""
|
317
|
+
cursor = self.conn.execute("""
|
318
|
+
SELECT
|
319
|
+
t.*,
|
320
|
+
b.name as branch_name,
|
321
|
+
b.schema_version,
|
322
|
+
b.materialized as branch_materialized,
|
323
|
+
d.name as database_name,
|
324
|
+
d.materialized as database_materialized
|
325
|
+
FROM tenants t
|
326
|
+
JOIN branches b ON t.branch_id = b.id
|
327
|
+
JOIN databases d ON b.database_id = d.id
|
328
|
+
WHERE d.name = ? AND b.name = ? AND t.name = ?
|
329
|
+
""", (database_name, branch_name, tenant_name))
|
330
|
+
row = cursor.fetchone()
|
331
|
+
return dict(row) if row else None
|
332
|
+
|
333
|
+
# Utility methods
|
334
|
+
def copy_tenants_to_branch(self, source_branch_id: str, target_branch_id: str,
|
335
|
+
as_lazy: bool = True) -> int:
|
336
|
+
"""Copy all tenants from source branch to target branch.
|
337
|
+
|
338
|
+
Args:
|
339
|
+
source_branch_id: ID of source branch
|
340
|
+
target_branch_id: ID of target branch
|
341
|
+
as_lazy: If True, copied tenants are marked as not materialized
|
342
|
+
|
343
|
+
Returns:
|
344
|
+
Number of tenants copied
|
345
|
+
"""
|
346
|
+
with self.conn:
|
347
|
+
# Get all tenants from source branch
|
348
|
+
cursor = self.conn.execute("""
|
349
|
+
SELECT name, metadata FROM tenants
|
350
|
+
WHERE branch_id = ?
|
351
|
+
""", (source_branch_id,))
|
352
|
+
|
353
|
+
tenants = cursor.fetchall()
|
354
|
+
|
355
|
+
for tenant in tenants:
|
356
|
+
tenant_id = str(uuid.uuid4())
|
357
|
+
self.conn.execute("""
|
358
|
+
INSERT INTO tenants (id, branch_id, name, materialized, metadata)
|
359
|
+
VALUES (?, ?, ?, ?, ?)
|
360
|
+
""", (tenant_id, target_branch_id, tenant['name'],
|
361
|
+
not as_lazy, tenant['metadata']))
|
362
|
+
|
363
|
+
return len(tenants)
|
364
|
+
|
365
|
+
def close(self):
|
366
|
+
"""Close the database connection."""
|
367
|
+
if self.conn:
|
368
|
+
self.conn.close()
|
369
|
+
|
370
|
+
def __enter__(self):
|
371
|
+
"""Context manager entry."""
|
372
|
+
return self
|
373
|
+
|
374
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
375
|
+
"""Context manager exit."""
|
376
|
+
self.close()
|
cinchdb/managers/branch.py
CHANGED
@@ -2,8 +2,9 @@
|
|
2
2
|
|
3
3
|
import json
|
4
4
|
import shutil
|
5
|
+
import uuid
|
5
6
|
from pathlib import Path
|
6
|
-
from typing import List, Dict, Any
|
7
|
+
from typing import List, Dict, Any, Optional
|
7
8
|
from datetime import datetime, timezone
|
8
9
|
|
9
10
|
from cinchdb.models import Branch
|
@@ -13,6 +14,8 @@ from cinchdb.core.path_utils import (
|
|
13
14
|
list_branches,
|
14
15
|
)
|
15
16
|
from cinchdb.utils.name_validator import validate_name
|
17
|
+
from cinchdb.infrastructure.metadata_db import MetadataDB
|
18
|
+
from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
|
16
19
|
|
17
20
|
|
18
21
|
class BranchManager:
|
@@ -28,6 +31,27 @@ class BranchManager:
|
|
28
31
|
self.project_root = Path(project_root)
|
29
32
|
self.database = database
|
30
33
|
self.db_path = get_database_path(self.project_root, database)
|
34
|
+
|
35
|
+
# Lazy-initialized pooled connection
|
36
|
+
self._metadata_db = None
|
37
|
+
self.database_id = None
|
38
|
+
|
39
|
+
def _ensure_initialized(self) -> None:
|
40
|
+
"""Ensure metadata connection and IDs are initialized."""
|
41
|
+
if self._metadata_db is None:
|
42
|
+
self._metadata_db = get_metadata_db(self.project_root)
|
43
|
+
|
44
|
+
# Initialize database ID on first access
|
45
|
+
if self.database_id is None:
|
46
|
+
db_info = self._metadata_db.get_database(self.database)
|
47
|
+
if db_info:
|
48
|
+
self.database_id = db_info['id']
|
49
|
+
|
50
|
+
@property
|
51
|
+
def metadata_db(self) -> MetadataDB:
|
52
|
+
"""Get metadata database connection (lazy-initialized from pool)."""
|
53
|
+
self._ensure_initialized()
|
54
|
+
return self._metadata_db
|
31
55
|
|
32
56
|
def list_branches(self) -> List[Branch]:
|
33
57
|
"""List all branches in the database.
|
@@ -67,30 +91,84 @@ class BranchManager:
|
|
67
91
|
"""
|
68
92
|
# Validate new branch name
|
69
93
|
validate_name(new_branch_name, "branch")
|
70
|
-
|
71
|
-
#
|
72
|
-
|
94
|
+
|
95
|
+
# Ensure initialization
|
96
|
+
self._ensure_initialized()
|
97
|
+
|
98
|
+
if not self.database_id:
|
99
|
+
raise ValueError(f"Database '{self.database}' not found in metadata")
|
100
|
+
|
101
|
+
# Validate source branch exists in metadata
|
102
|
+
source_branch_info = self.metadata_db.get_branch(self.database_id, source_branch)
|
103
|
+
if not source_branch_info:
|
73
104
|
raise ValueError(f"Source branch '{source_branch}' does not exist")
|
74
105
|
|
75
106
|
# Validate new branch doesn't exist
|
76
|
-
|
107
|
+
existing_branch = self.metadata_db.get_branch(self.database_id, new_branch_name)
|
108
|
+
if existing_branch:
|
77
109
|
raise ValueError(f"Branch '{new_branch_name}' already exists")
|
78
110
|
|
79
|
-
#
|
111
|
+
# Create branch ID
|
112
|
+
branch_id = str(uuid.uuid4())
|
113
|
+
|
114
|
+
# Get source branch schema version
|
115
|
+
schema_version = source_branch_info['schema_version'] if source_branch_info else "v1.0.0"
|
116
|
+
|
117
|
+
# Create branch in metadata
|
118
|
+
metadata = {
|
119
|
+
"created_at": datetime.now(timezone.utc).isoformat(),
|
120
|
+
"copied_from": source_branch,
|
121
|
+
}
|
122
|
+
self.metadata_db.create_branch(
|
123
|
+
branch_id, self.database_id, new_branch_name,
|
124
|
+
parent_branch=source_branch,
|
125
|
+
schema_version=schema_version,
|
126
|
+
metadata=metadata
|
127
|
+
)
|
128
|
+
|
129
|
+
# Copy all tenant entries from source branch to new branch
|
130
|
+
# Tenants are branch-specific, so each branch needs its own tenant entries
|
131
|
+
source_tenants = self.metadata_db.list_tenants(source_branch_info['id'])
|
132
|
+
for tenant in source_tenants:
|
133
|
+
new_tenant_id = str(uuid.uuid4())
|
134
|
+
tenant_metadata = json.loads(tenant['metadata']) if tenant['metadata'] else {}
|
135
|
+
tenant_metadata['copied_from'] = source_branch
|
136
|
+
|
137
|
+
# Create tenant in new branch, preserving materialization status and shard info
|
138
|
+
self.metadata_db.create_tenant(
|
139
|
+
new_tenant_id, branch_id, tenant['name'], tenant['shard'],
|
140
|
+
metadata=tenant_metadata
|
141
|
+
)
|
142
|
+
|
143
|
+
# Preserve materialization status from source branch
|
144
|
+
if tenant['materialized']:
|
145
|
+
self.metadata_db.mark_tenant_materialized(new_tenant_id)
|
146
|
+
|
147
|
+
# Ensure __empty__ tenant exists (in case source branch didn't have it)
|
148
|
+
if not any(t['name'] == '__empty__' for t in source_tenants):
|
149
|
+
import hashlib
|
150
|
+
empty_shard = hashlib.sha256("__empty__".encode('utf-8')).hexdigest()[:2]
|
151
|
+
empty_tenant_id = str(uuid.uuid4())
|
152
|
+
self.metadata_db.create_tenant(
|
153
|
+
empty_tenant_id, branch_id, "__empty__", empty_shard,
|
154
|
+
metadata={"system": True, "description": "Template for lazy tenants"}
|
155
|
+
)
|
156
|
+
|
157
|
+
# Copy entire branch directory (branches should copy ALL files including tenants)
|
80
158
|
source_path = get_branch_path(self.project_root, self.database, source_branch)
|
81
159
|
new_path = get_branch_path(self.project_root, self.database, new_branch_name)
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
#
|
93
|
-
|
160
|
+
|
161
|
+
if source_path.exists():
|
162
|
+
shutil.copytree(source_path, new_path)
|
163
|
+
|
164
|
+
# Update branch metadata file
|
165
|
+
fs_metadata = self.get_branch_metadata(new_branch_name)
|
166
|
+
fs_metadata["parent_branch"] = source_branch
|
167
|
+
fs_metadata["created_at"] = datetime.now(timezone.utc).isoformat()
|
168
|
+
self.update_branch_metadata(new_branch_name, fs_metadata)
|
169
|
+
|
170
|
+
# Mark branch as materialized (always true now)
|
171
|
+
self.metadata_db.mark_branch_materialized(branch_id)
|
94
172
|
|
95
173
|
return Branch(
|
96
174
|
name=new_branch_name,
|
@@ -112,14 +190,31 @@ class BranchManager:
|
|
112
190
|
# Can't delete main branch
|
113
191
|
if branch_name == "main":
|
114
192
|
raise ValueError("Cannot delete the main branch")
|
115
|
-
|
116
|
-
#
|
117
|
-
|
193
|
+
|
194
|
+
# Ensure initialization
|
195
|
+
self._ensure_initialized()
|
196
|
+
|
197
|
+
if not self.database_id:
|
198
|
+
raise ValueError(f"Database '{self.database}' not found in metadata")
|
199
|
+
|
200
|
+
# Check if branch exists in metadata
|
201
|
+
branch_info = self.metadata_db.get_branch(self.database_id, branch_name)
|
202
|
+
if not branch_info:
|
118
203
|
raise ValueError(f"Branch '{branch_name}' does not exist")
|
204
|
+
|
205
|
+
# NOTE: We delete from metadata first because it's better to have a scared, lost file than a zombie branch
|
206
|
+
|
207
|
+
# Delete from metadata (cascade deletes will handle tenants)
|
208
|
+
with self.metadata_db.conn:
|
209
|
+
self.metadata_db.conn.execute(
|
210
|
+
"DELETE FROM branches WHERE id = ?",
|
211
|
+
(branch_info['id'],)
|
212
|
+
)
|
119
213
|
|
120
|
-
# Delete branch directory
|
214
|
+
# Delete branch directory if it exists
|
121
215
|
branch_path = get_branch_path(self.project_root, self.database, branch_name)
|
122
|
-
|
216
|
+
if branch_path.exists():
|
217
|
+
shutil.rmtree(branch_path)
|
123
218
|
|
124
219
|
def get_branch_metadata(self, branch_name: str) -> Dict[str, Any]:
|
125
220
|
"""Get metadata for a branch.
|
@@ -168,3 +263,4 @@ class BranchManager:
|
|
168
263
|
True if branch exists, False otherwise
|
169
264
|
"""
|
170
265
|
return branch_name in list_branches(self.project_root, self.database)
|
266
|
+
|
@@ -64,6 +64,8 @@ class ChangeApplier:
|
|
64
64
|
def apply_change(self, change_id: str) -> None:
|
65
65
|
"""Apply a single change to all tenants atomically with snapshot-based rollback.
|
66
66
|
|
67
|
+
For lazy tenants, skip applying changes - they will inherit changes from the __empty__ tenant later.
|
68
|
+
|
67
69
|
Args:
|
68
70
|
change_id: ID of the change to apply
|
69
71
|
|
@@ -81,20 +83,35 @@ class ChangeApplier:
|
|
81
83
|
logger.info(f"Change {change_id} already applied")
|
82
84
|
return
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
+
# Ensure __empty__ tenant exists and is materialized
|
87
|
+
# This is critical for schema changes as it's the template for lazy tenants
|
88
|
+
self.tenant_manager._ensure_empty_tenant()
|
86
89
|
|
87
|
-
|
88
|
-
|
90
|
+
backup_dir = self._get_backup_dir(change.id)
|
91
|
+
# Include system tenants (like __empty__) when applying schema changes
|
92
|
+
all_tenants = self.tenant_manager.list_tenants(include_system=True)
|
93
|
+
|
94
|
+
# Filter to only materialized tenants (those with actual .db files)
|
95
|
+
# Schema changes can only be applied to materialized tenants
|
96
|
+
# Lazy tenants will inherit the schema from __empty__ when materialized
|
97
|
+
materialized_tenants = []
|
98
|
+
for tenant in all_tenants:
|
99
|
+
# Always include __empty__ (schema template) or check if materialized
|
100
|
+
if tenant.name == "__empty__" or not self.tenant_manager.is_tenant_lazy(tenant.name):
|
101
|
+
materialized_tenants.append(tenant)
|
102
|
+
|
103
|
+
if not materialized_tenants:
|
104
|
+
# No materialized tenants, just mark as applied
|
105
|
+
# The schema change will be in __empty__ for future lazy tenant materialization
|
89
106
|
self.change_tracker.mark_change_applied(change_id)
|
90
107
|
return
|
91
108
|
|
92
|
-
logger.info(f"Applying change {change_id} to {len(tenants)}
|
109
|
+
logger.info(f"Applying change {change_id} to {len(materialized_tenants)} materialized tenants (out of {len(all_tenants)} total)...")
|
93
110
|
|
94
111
|
try:
|
95
|
-
# Phase 1: Create snapshots
|
112
|
+
# Phase 1: Create snapshots (only for materialized tenants)
|
96
113
|
logger.info("Creating database snapshots...")
|
97
|
-
self._create_snapshots(
|
114
|
+
self._create_snapshots(materialized_tenants, backup_dir)
|
98
115
|
|
99
116
|
# Phase 2: Enter maintenance mode to block writes
|
100
117
|
logger.info("Entering maintenance mode for schema update...")
|
@@ -104,7 +121,7 @@ class ChangeApplier:
|
|
104
121
|
# Track which tenants we've applied to
|
105
122
|
applied_tenants = []
|
106
123
|
|
107
|
-
for tenant in
|
124
|
+
for tenant in materialized_tenants:
|
108
125
|
try:
|
109
126
|
self._apply_change_to_tenant(change, tenant.name)
|
110
127
|
applied_tenants.append(tenant.name)
|
@@ -127,7 +144,7 @@ class ChangeApplier:
|
|
127
144
|
self._cleanup_snapshots(backup_dir)
|
128
145
|
|
129
146
|
logger.info(
|
130
|
-
f"Schema update complete. Applied change {change_id} to {len(
|
147
|
+
f"Schema update complete. Applied change {change_id} to {len(materialized_tenants)} materialized tenants"
|
131
148
|
)
|
132
149
|
|
133
150
|
except Exception:
|
@@ -136,12 +153,12 @@ class ChangeApplier:
|
|
136
153
|
raise
|
137
154
|
|
138
155
|
except Exception as e:
|
139
|
-
# Rollback all tenants
|
156
|
+
# Rollback all materialized tenants
|
140
157
|
logger.error(f"Change {change_id} failed: {e}")
|
141
|
-
logger.info("Rolling back all tenants to snapshot...")
|
158
|
+
logger.info("Rolling back all materialized tenants to snapshot...")
|
142
159
|
|
143
|
-
# Restore all tenants from snapshots
|
144
|
-
self._restore_all_snapshots(
|
160
|
+
# Restore all materialized tenants from snapshots
|
161
|
+
self._restore_all_snapshots(materialized_tenants, backup_dir)
|
145
162
|
|
146
163
|
# Clean up backup directory
|
147
164
|
self._cleanup_snapshots(backup_dir)
|