dockflow-core 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dockflow_core-0.2.0/LICENSE +21 -0
- dockflow_core-0.2.0/PKG-INFO +29 -0
- dockflow_core-0.2.0/README.md +18 -0
- dockflow_core-0.2.0/pyproject.toml +16 -0
- dockflow_core-0.2.0/src/dockflow_core/__init__.py +8 -0
- dockflow_core-0.2.0/src/dockflow_core/canonical.py +114 -0
- dockflow_core-0.2.0/src/dockflow_core/coordination.py +22 -0
- dockflow_core-0.2.0/src/dockflow_core/database.py +501 -0
- dockflow_core-0.2.0/src/dockflow_core/errors.py +27 -0
- dockflow_core-0.2.0/src/dockflow_core/filesystem.py +56 -0
- dockflow_core-0.2.0/src/dockflow_core/locking.py +26 -0
- dockflow_core-0.2.0/src/dockflow_core/py.typed +0 -0
- dockflow_core-0.2.0/src/dockflow_core/schema.py +35 -0
- dockflow_core-0.2.0/src/dockflow_core/time.py +7 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dockflow contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dockflow-core
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Domain-independent shared primitives for Dockflow
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: psycopg[binary]>=3.3.6,<4 ; extra == 'postgresql'
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Provides-Extra: postgresql
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# Dockflow Core
|
|
13
|
+
|
|
14
|
+
Domain-independent primitives shared by Dockflow CLI, Pipelines and Artifacts:
|
|
15
|
+
SQLite/PostgreSQL adapters, caller-defined schema initialization/versioning,
|
|
16
|
+
generic locks, common errors, portable JSON encoding, stream hashing and durable
|
|
17
|
+
filesystem operations. Core imports no consuming package and owns no domain
|
|
18
|
+
schema, models, queries or persistence policy.
|
|
19
|
+
|
|
20
|
+
Install `dockflow-core==0.2.0`, or `dockflow-core[postgresql]==0.2.0` for
|
|
21
|
+
PostgreSQL support. The SQLite implementation uses the Python standard library.
|
|
22
|
+
Optional drivers and connections are initialized only by explicit operations.
|
|
23
|
+
|
|
24
|
+
Consumers supply `Schema` definitions and retain their own queries and policies.
|
|
25
|
+
Several independently versioned schemas can share one physical database.
|
|
26
|
+
|
|
27
|
+
This package follows the shared Dockflow application version and MIT license.
|
|
28
|
+
Development includes AI assistance. See the repository documentation for
|
|
29
|
+
configuration and deployment requirements.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Dockflow Core
|
|
2
|
+
|
|
3
|
+
Domain-independent primitives shared by Dockflow CLI, Pipelines and Artifacts:
|
|
4
|
+
SQLite/PostgreSQL adapters, caller-defined schema initialization/versioning,
|
|
5
|
+
generic locks, common errors, portable JSON encoding, stream hashing and durable
|
|
6
|
+
filesystem operations. Core imports no consuming package and owns no domain
|
|
7
|
+
schema, models, queries or persistence policy.
|
|
8
|
+
|
|
9
|
+
Install `dockflow-core==0.2.0`, or `dockflow-core[postgresql]==0.2.0` for
|
|
10
|
+
PostgreSQL support. The SQLite implementation uses the Python standard library.
|
|
11
|
+
Optional drivers and connections are initialized only by explicit operations.
|
|
12
|
+
|
|
13
|
+
Consumers supply `Schema` definitions and retain their own queries and policies.
|
|
14
|
+
Several independently versioned schemas can share one physical database.
|
|
15
|
+
|
|
16
|
+
This package follows the shared Dockflow application version and MIT license.
|
|
17
|
+
Development includes AI assistance. See the repository documentation for
|
|
18
|
+
configuration and deployment requirements.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dockflow-core"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Domain-independent shared primitives for Dockflow"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
dependencies = []
|
|
10
|
+
|
|
11
|
+
[project.optional-dependencies]
|
|
12
|
+
postgresql = ["psycopg[binary]>=3.3.6,<4"]
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["uv_build>=0.11.2,<0.12"]
|
|
16
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Domain-independent Dockflow primitives; imports perform no external I/O."""
|
|
2
|
+
|
|
3
|
+
from .coordination import LockScope
|
|
4
|
+
from .database import Database, database
|
|
5
|
+
from .errors import DockflowError
|
|
6
|
+
from .schema import Schema
|
|
7
|
+
|
|
8
|
+
__all__ = ["Database", "DockflowError", "LockScope", "Schema", "database"]
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Portable JSON validation, serialization and deterministic typed encoding."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import struct
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
MAX_INTEGER = 2**53 - 1
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _string(value: str) -> str:
|
|
15
|
+
try:
|
|
16
|
+
value.encode("utf-8", errors="strict")
|
|
17
|
+
except UnicodeError as exc:
|
|
18
|
+
raise ValueError("JSON strings cannot contain unpaired surrogates") from exc
|
|
19
|
+
return value
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def canonical_tree(
|
|
23
|
+
value: Any, *, typed_keys: bool = False, strict_types: bool = False
|
|
24
|
+
) -> list[Any]:
|
|
25
|
+
"""Encode a portable JSON value without losing its numeric type or -0.0."""
|
|
26
|
+
|
|
27
|
+
def is_kind(kind):
|
|
28
|
+
return type(value) is kind if strict_types else isinstance(value, kind)
|
|
29
|
+
|
|
30
|
+
def encode(item):
|
|
31
|
+
return canonical_tree(item, typed_keys=typed_keys, strict_types=strict_types)
|
|
32
|
+
|
|
33
|
+
if value is None:
|
|
34
|
+
return ["null"]
|
|
35
|
+
if is_kind(bool):
|
|
36
|
+
return ["bool", value]
|
|
37
|
+
if is_kind(int):
|
|
38
|
+
if abs(value) > MAX_INTEGER:
|
|
39
|
+
raise ValueError("JSON integer exceeds the portable 53-bit range")
|
|
40
|
+
return ["int", str(value)]
|
|
41
|
+
if is_kind(float):
|
|
42
|
+
if not math.isfinite(value):
|
|
43
|
+
raise ValueError("JSON numbers must be finite")
|
|
44
|
+
return ["float64", struct.pack(">d", value).hex()]
|
|
45
|
+
if is_kind(str):
|
|
46
|
+
return ["string", _string(value)]
|
|
47
|
+
if is_kind(list):
|
|
48
|
+
return ["array", [encode(item) for item in value]]
|
|
49
|
+
if is_kind(dict):
|
|
50
|
+
if any(
|
|
51
|
+
(type(key) is not str if strict_types else not isinstance(key, str))
|
|
52
|
+
for key in value
|
|
53
|
+
):
|
|
54
|
+
raise ValueError("JSON object keys must be strings")
|
|
55
|
+
return [
|
|
56
|
+
"object",
|
|
57
|
+
[
|
|
58
|
+
[encode(key) if typed_keys else _string(key), encode(value[key])]
|
|
59
|
+
for key in sorted(value)
|
|
60
|
+
],
|
|
61
|
+
]
|
|
62
|
+
raise ValueError(f"Unsupported JSON value type: {type(value).__name__}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def validate_json(value: Any) -> None:
|
|
66
|
+
canonical_tree(value)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def canonical_bytes(value: Any) -> bytes:
|
|
70
|
+
return json.dumps(
|
|
71
|
+
canonical_tree(value), ensure_ascii=False, separators=(",", ":")
|
|
72
|
+
).encode("utf-8")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def digest(value: Any) -> str:
|
|
76
|
+
return hashlib.sha256(canonical_bytes(value)).hexdigest()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def json_bytes(value: Any) -> bytes:
|
|
80
|
+
"""Serialize validated ordinary JSON, preserving portable parameter types."""
|
|
81
|
+
validate_json(value)
|
|
82
|
+
return json.dumps(
|
|
83
|
+
value,
|
|
84
|
+
ensure_ascii=False,
|
|
85
|
+
allow_nan=False,
|
|
86
|
+
separators=(",", ":"),
|
|
87
|
+
sort_keys=True,
|
|
88
|
+
).encode("utf-8")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def loads(source: str | bytes) -> Any:
|
|
92
|
+
"""Strict JSON parsing shared by durable records and external boundaries."""
|
|
93
|
+
|
|
94
|
+
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
95
|
+
result: dict[str, Any] = {}
|
|
96
|
+
for key, value in items:
|
|
97
|
+
if key in result:
|
|
98
|
+
raise ValueError(f"Duplicate JSON object key: {key}")
|
|
99
|
+
result[key] = value
|
|
100
|
+
return result
|
|
101
|
+
|
|
102
|
+
def constant(value: str) -> None:
|
|
103
|
+
raise ValueError(f"Non-finite JSON number: {value}")
|
|
104
|
+
|
|
105
|
+
result = json.loads(source, object_pairs_hook=pairs, parse_constant=constant)
|
|
106
|
+
validate_json(result)
|
|
107
|
+
return result
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def stable_json_bytes(value: Any) -> bytes:
|
|
111
|
+
"""Sorted finite JSON with ASCII escaping, without a typed identity envelope."""
|
|
112
|
+
return json.dumps(
|
|
113
|
+
value, sort_keys=True, separators=(",", ":"), allow_nan=False
|
|
114
|
+
).encode()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Generic lock scopes; consumers supply namespace and identity semantics."""
|
|
2
|
+
|
|
3
|
+
from .database import Database
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class LockScope:
|
|
7
|
+
def __init__(self, database: Database, namespace: str, identity: str):
|
|
8
|
+
self.database = database
|
|
9
|
+
self.key = database.lock_key(namespace, identity)
|
|
10
|
+
self.mutation_key = database.lock_key(namespace + ":mutation", identity)
|
|
11
|
+
|
|
12
|
+
def gate(self, connection) -> None:
|
|
13
|
+
self.database.transaction_lock(connection, self.key, shared=True)
|
|
14
|
+
|
|
15
|
+
def serialize(self, connection) -> None:
|
|
16
|
+
self.database.transaction_lock(connection, self.mutation_key)
|
|
17
|
+
|
|
18
|
+
def try_exclusive(self, connection) -> bool:
|
|
19
|
+
return self.database.try_lock(connection, self.key)
|
|
20
|
+
|
|
21
|
+
def release_exclusive(self, connection) -> None:
|
|
22
|
+
self.database.unlock(connection, self.key)
|
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
"""Domain-independent database backend boundary.
|
|
2
|
+
|
|
3
|
+
Connections, SQL dialect, durability, schema and driver failure classification
|
|
4
|
+
live here. Callers retain their transaction/retry policies and decide whether an
|
|
5
|
+
uncertain commit invalidates their session or an idempotent operation may retry.
|
|
6
|
+
No connection is shared by workers.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import errno
|
|
13
|
+
import hashlib
|
|
14
|
+
import importlib
|
|
15
|
+
import math
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import sqlite3
|
|
19
|
+
import stat
|
|
20
|
+
import time
|
|
21
|
+
from abc import ABC, abstractmethod
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from .errors import DockflowError
|
|
26
|
+
from .locking import owner_lock as _owner_lock
|
|
27
|
+
from .schema import Schema
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class OwnerConflict(Exception):
|
|
31
|
+
"""Another process holds the requested database lock."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Database(ABC):
|
|
35
|
+
backend: str
|
|
36
|
+
binary_type: str
|
|
37
|
+
write_begin: str
|
|
38
|
+
snapshot_begin: str
|
|
39
|
+
row_lock: str
|
|
40
|
+
clock_expression: str
|
|
41
|
+
|
|
42
|
+
def __init__(self, options: dict):
|
|
43
|
+
self.options = dict(options)
|
|
44
|
+
|
|
45
|
+
@abstractmethod
|
|
46
|
+
def connect(
|
|
47
|
+
self, *, readonly=False, mapping=False, timeout=5.0, create=True
|
|
48
|
+
) -> Any:
|
|
49
|
+
"""Open an autocommit connection, without opening a transaction."""
|
|
50
|
+
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def prepare(self) -> None:
|
|
53
|
+
"""Prepare durable storage only for an authorized mutating operation."""
|
|
54
|
+
|
|
55
|
+
@abstractmethod
|
|
56
|
+
def tables(self, connection) -> set[str]: ...
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def indexes(self, connection) -> set[str]: ...
|
|
60
|
+
|
|
61
|
+
@abstractmethod
|
|
62
|
+
def owner_connection(
|
|
63
|
+
self, lock_id: int, *, exclusive: bool = False
|
|
64
|
+
) -> tuple[Any, Any]: ...
|
|
65
|
+
|
|
66
|
+
supports_shared_owners = False
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def lock_key(namespace: str, identity: str) -> int:
|
|
70
|
+
"""Derive a stable signed advisory key from caller-owned names."""
|
|
71
|
+
encoded_namespace = namespace.encode()
|
|
72
|
+
framed = (
|
|
73
|
+
str(len(encoded_namespace)).encode()
|
|
74
|
+
+ b":"
|
|
75
|
+
+ encoded_namespace
|
|
76
|
+
+ identity.encode()
|
|
77
|
+
)
|
|
78
|
+
return int.from_bytes(
|
|
79
|
+
hashlib.sha256(framed).digest()[:8],
|
|
80
|
+
"big",
|
|
81
|
+
signed=True,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def transaction_lock(self, connection, key: int, *, shared=False) -> None:
|
|
85
|
+
"""SQLite's write transaction already serializes writers."""
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
def try_lock(self, connection, key: int, *, shared=False) -> bool:
|
|
89
|
+
return True
|
|
90
|
+
|
|
91
|
+
def unlock(self, connection, key: int, *, shared=False) -> None:
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
def sql(self, statement: str) -> str:
|
|
95
|
+
return statement
|
|
96
|
+
|
|
97
|
+
def execute(self, connection, statement: str, parameters=()):
|
|
98
|
+
return connection.execute(self.sql(statement), parameters)
|
|
99
|
+
|
|
100
|
+
def json_text(self, column: str, path: str) -> str:
|
|
101
|
+
"""Internal scalar query view; never coerce canonical stored JSON."""
|
|
102
|
+
import re
|
|
103
|
+
|
|
104
|
+
if not re.fullmatch(
|
|
105
|
+
r"[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)?", column
|
|
106
|
+
) or not re.fullmatch(r"[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)*", path):
|
|
107
|
+
raise ValueError("Invalid internal JSON query path")
|
|
108
|
+
if self.backend == "postgresql":
|
|
109
|
+
return f"({column}::jsonb #>> '{{{path.replace('.', ',')}}}')"
|
|
110
|
+
return f"CAST(json_extract(CAST({column} AS TEXT), '$.{path}') AS TEXT)"
|
|
111
|
+
|
|
112
|
+
def text_order(self, expression: str) -> str:
|
|
113
|
+
collation = '"C"' if self.backend == "postgresql" else "BINARY"
|
|
114
|
+
return f"({expression} COLLATE {collation})"
|
|
115
|
+
|
|
116
|
+
def begin(self, connection, *, readonly=False):
|
|
117
|
+
connection.execute(self.snapshot_begin if readonly else self.write_begin)
|
|
118
|
+
|
|
119
|
+
def schema_exists(self, connection, schema: Schema) -> bool:
|
|
120
|
+
if "dockflow_schema_components" not in self.tables(connection):
|
|
121
|
+
return False
|
|
122
|
+
return (
|
|
123
|
+
self.execute(
|
|
124
|
+
connection,
|
|
125
|
+
"SELECT version FROM dockflow_schema_components WHERE name=?",
|
|
126
|
+
(schema.name,),
|
|
127
|
+
).fetchone()
|
|
128
|
+
is not None
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def validate_schema(self, connection, schema: Schema) -> None:
|
|
132
|
+
if not self.schema_exists(connection, schema):
|
|
133
|
+
raise DockflowError("Database schema is missing", "CONFLICT", 409)
|
|
134
|
+
row = self.execute(
|
|
135
|
+
connection,
|
|
136
|
+
"SELECT version FROM dockflow_schema_components WHERE name=?",
|
|
137
|
+
(schema.name,),
|
|
138
|
+
).fetchone()
|
|
139
|
+
version = row["version"] if isinstance(row, dict) else row[0]
|
|
140
|
+
if version != schema.version:
|
|
141
|
+
raise DockflowError("Unsupported database schema version", "CONFLICT", 409)
|
|
142
|
+
if not schema.tables <= self.tables(
|
|
143
|
+
connection
|
|
144
|
+
) or not schema.indexes <= self.indexes(connection):
|
|
145
|
+
raise DockflowError(
|
|
146
|
+
"Database schema is incomplete; restore a verified backup",
|
|
147
|
+
"CONFLICT",
|
|
148
|
+
409,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
def schema_lock(self, connection):
|
|
152
|
+
"""The SQLite write transaction already serializes schema creation."""
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
def initialize_schema(self, connection, schema: Schema) -> None:
|
|
156
|
+
"""Initialize only the supplied definition inside a caller write transaction."""
|
|
157
|
+
self.schema_lock(connection)
|
|
158
|
+
if self.schema_exists(connection, schema):
|
|
159
|
+
self.validate_schema(connection, schema)
|
|
160
|
+
return
|
|
161
|
+
tables = self.tables(connection)
|
|
162
|
+
registry = "dockflow_schema_components"
|
|
163
|
+
if schema.tables & tables or (
|
|
164
|
+
self.backend == "sqlite" and tables and registry not in tables
|
|
165
|
+
):
|
|
166
|
+
raise DockflowError(
|
|
167
|
+
"Unregistered database objects conflict; select fresh state",
|
|
168
|
+
"CONFLICT",
|
|
169
|
+
409,
|
|
170
|
+
)
|
|
171
|
+
connection.execute(
|
|
172
|
+
"CREATE TABLE IF NOT EXISTS dockflow_schema_components (name TEXT PRIMARY KEY, version INTEGER NOT NULL CHECK(version>0))"
|
|
173
|
+
)
|
|
174
|
+
for statement in schema.statements:
|
|
175
|
+
connection.execute(statement.format(binary=self.binary_type))
|
|
176
|
+
if schema.initialize is not None:
|
|
177
|
+
schema.initialize(self, connection)
|
|
178
|
+
self.execute(
|
|
179
|
+
connection,
|
|
180
|
+
"INSERT INTO dockflow_schema_components VALUES (?,?)",
|
|
181
|
+
(schema.name, schema.version),
|
|
182
|
+
)
|
|
183
|
+
self.validate_schema(connection, schema)
|
|
184
|
+
|
|
185
|
+
@contextlib.contextmanager
|
|
186
|
+
def snapshot(self):
|
|
187
|
+
connection = None
|
|
188
|
+
try:
|
|
189
|
+
connection = self.connect(readonly=True)
|
|
190
|
+
if connection is None:
|
|
191
|
+
raise DockflowError("Database snapshot is missing", "NOT_FOUND", 404)
|
|
192
|
+
self.begin(connection, readonly=True)
|
|
193
|
+
yield connection
|
|
194
|
+
except DockflowError:
|
|
195
|
+
raise
|
|
196
|
+
except Exception:
|
|
197
|
+
raise DockflowError(
|
|
198
|
+
"Cannot read Dockflow database snapshot", "UNAVAILABLE", 503
|
|
199
|
+
) from None
|
|
200
|
+
finally:
|
|
201
|
+
if connection is not None:
|
|
202
|
+
connection.close()
|
|
203
|
+
|
|
204
|
+
@staticmethod
|
|
205
|
+
def unique_error(error) -> bool:
|
|
206
|
+
return (
|
|
207
|
+
isinstance(error, sqlite3.IntegrityError)
|
|
208
|
+
or getattr(error, "sqlstate", None) == "23505"
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
@staticmethod
|
|
212
|
+
def retryable_error(error) -> bool:
|
|
213
|
+
code = getattr(error, "sqlstate", None)
|
|
214
|
+
return (
|
|
215
|
+
(
|
|
216
|
+
isinstance(error, sqlite3.OperationalError)
|
|
217
|
+
and getattr(error, "sqlite_errorcode", 0) & 0xFF
|
|
218
|
+
in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}
|
|
219
|
+
)
|
|
220
|
+
or code in {"40001", "40P01", "55P03", "57014"}
|
|
221
|
+
or (
|
|
222
|
+
(code is None or code.startswith("08"))
|
|
223
|
+
and type(error).__name__ in {"OperationalError", "InterfaceError"}
|
|
224
|
+
and not isinstance(error, sqlite3.Error)
|
|
225
|
+
)
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class SQLiteDatabase(Database):
|
|
230
|
+
backend = "sqlite"
|
|
231
|
+
binary_type = "BLOB"
|
|
232
|
+
write_begin = "BEGIN IMMEDIATE"
|
|
233
|
+
snapshot_begin = "BEGIN"
|
|
234
|
+
row_lock = ""
|
|
235
|
+
clock_expression = "(julianday('now') - 2440587.5) * 86400.0"
|
|
236
|
+
|
|
237
|
+
def connect(self, *, readonly=False, mapping=False, timeout=5.0, create=True):
|
|
238
|
+
# Several publishers may initialize a fresh SQLite file simultaneously.
|
|
239
|
+
# Switching to WAL can report BUSY without honoring busy_timeout. Retry
|
|
240
|
+
# only connection setup, before the caller has executed any transaction.
|
|
241
|
+
deadline = time.monotonic() + timeout
|
|
242
|
+
while True:
|
|
243
|
+
try:
|
|
244
|
+
return self._connect_once(
|
|
245
|
+
readonly=readonly, mapping=mapping, timeout=timeout, create=create
|
|
246
|
+
)
|
|
247
|
+
except sqlite3.OperationalError as error:
|
|
248
|
+
if not self.retryable_error(error) or time.monotonic() >= deadline:
|
|
249
|
+
raise
|
|
250
|
+
time.sleep(0.01)
|
|
251
|
+
|
|
252
|
+
def _connect_once(self, *, readonly=False, mapping=False, timeout=5.0, create=True):
|
|
253
|
+
path = Path(self.options["database"]).absolute()
|
|
254
|
+
if not path.exists() and (readonly or not create):
|
|
255
|
+
if readonly:
|
|
256
|
+
return None
|
|
257
|
+
raise DockflowError(
|
|
258
|
+
"Dockflow database is uninitialized", "UNAVAILABLE", 503
|
|
259
|
+
)
|
|
260
|
+
connection = sqlite3.connect(
|
|
261
|
+
path.as_uri()
|
|
262
|
+
+ ("?mode=ro" if readonly else "?mode=rwc" if create else "?mode=rw"),
|
|
263
|
+
uri=True,
|
|
264
|
+
timeout=timeout,
|
|
265
|
+
isolation_level=None,
|
|
266
|
+
check_same_thread=False,
|
|
267
|
+
)
|
|
268
|
+
try:
|
|
269
|
+
if mapping:
|
|
270
|
+
connection.row_factory = sqlite3.Row
|
|
271
|
+
connection.execute("PRAGMA foreign_keys=ON")
|
|
272
|
+
if not readonly:
|
|
273
|
+
if connection.execute("PRAGMA journal_mode=WAL").fetchone()[0] != "wal":
|
|
274
|
+
raise DockflowError(
|
|
275
|
+
"Dockflow database requires SQLite WAL", "UNAVAILABLE", 503
|
|
276
|
+
)
|
|
277
|
+
connection.execute("PRAGMA synchronous=FULL")
|
|
278
|
+
return connection
|
|
279
|
+
except BaseException:
|
|
280
|
+
connection.close()
|
|
281
|
+
raise
|
|
282
|
+
|
|
283
|
+
def prepare(self):
|
|
284
|
+
from .filesystem import reject_symlinks
|
|
285
|
+
|
|
286
|
+
path = Path(self.options["database"])
|
|
287
|
+
try:
|
|
288
|
+
reject_symlinks(path)
|
|
289
|
+
except ValueError:
|
|
290
|
+
raise DockflowError("Database paths cannot contain symlinks") from None
|
|
291
|
+
reject_network_sqlite(path.absolute())
|
|
292
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
293
|
+
|
|
294
|
+
def tables(self, connection):
|
|
295
|
+
return {
|
|
296
|
+
row["name"] if isinstance(row, dict) else row[0]
|
|
297
|
+
for row in connection.execute(
|
|
298
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
|
299
|
+
).fetchall()
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
def indexes(self, connection):
|
|
303
|
+
return {
|
|
304
|
+
row[0]
|
|
305
|
+
for row in connection.execute(
|
|
306
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
307
|
+
).fetchall()
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
def owner_connection(self, lock_id, *, exclusive=False):
|
|
311
|
+
self.prepare()
|
|
312
|
+
path = Path(self.options["database"])
|
|
313
|
+
lock_path = (
|
|
314
|
+
path.with_suffix(path.suffix + ".owner") if os.name == "nt" else path
|
|
315
|
+
)
|
|
316
|
+
descriptor = os.open(
|
|
317
|
+
lock_path,
|
|
318
|
+
os.O_RDWR
|
|
319
|
+
| os.O_CREAT
|
|
320
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
321
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
322
|
+
| getattr(os, "O_CLOEXEC", 0),
|
|
323
|
+
0o600,
|
|
324
|
+
)
|
|
325
|
+
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
|
|
326
|
+
os.close(descriptor)
|
|
327
|
+
raise DockflowError(
|
|
328
|
+
"Database ownership requires a regular database file",
|
|
329
|
+
"UNAVAILABLE",
|
|
330
|
+
503,
|
|
331
|
+
)
|
|
332
|
+
stream = os.fdopen(descriptor, "r+b")
|
|
333
|
+
try:
|
|
334
|
+
try:
|
|
335
|
+
_owner_lock(stream, acquire=True)
|
|
336
|
+
except OSError as error:
|
|
337
|
+
if error.errno in {errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK}:
|
|
338
|
+
raise OwnerConflict from None
|
|
339
|
+
raise
|
|
340
|
+
return self.connect(), stream
|
|
341
|
+
except BaseException:
|
|
342
|
+
stream.close()
|
|
343
|
+
raise
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class PostgreSQLDatabase(Database):
|
|
347
|
+
supports_shared_owners = True
|
|
348
|
+
backend = "postgresql"
|
|
349
|
+
binary_type = "BYTEA"
|
|
350
|
+
write_begin = "BEGIN ISOLATION LEVEL READ COMMITTED"
|
|
351
|
+
snapshot_begin = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"
|
|
352
|
+
row_lock = " FOR UPDATE"
|
|
353
|
+
clock_expression = "EXTRACT(EPOCH FROM clock_timestamp())"
|
|
354
|
+
|
|
355
|
+
def connect(self, *, readonly=False, mapping=False, timeout=5.0, create=True):
|
|
356
|
+
try:
|
|
357
|
+
module = importlib.import_module("psycopg")
|
|
358
|
+
except ImportError:
|
|
359
|
+
raise DockflowError(
|
|
360
|
+
"Install dockflow-core[postgresql] for PostgreSQL",
|
|
361
|
+
"UNAVAILABLE",
|
|
362
|
+
503,
|
|
363
|
+
) from None
|
|
364
|
+
dsn = os.environ.get(self.options["dsn_env"])
|
|
365
|
+
if not dsn:
|
|
366
|
+
raise DockflowError(
|
|
367
|
+
"Database DSN environment variable is missing or empty",
|
|
368
|
+
"UNAVAILABLE",
|
|
369
|
+
503,
|
|
370
|
+
)
|
|
371
|
+
milliseconds = max(1, int(timeout * 1000))
|
|
372
|
+
return module.connect(
|
|
373
|
+
dsn,
|
|
374
|
+
autocommit=True,
|
|
375
|
+
connect_timeout=max(1, math.ceil(timeout)),
|
|
376
|
+
options=f"-c statement_timeout={milliseconds} -c lock_timeout={milliseconds} -c synchronous_commit=on",
|
|
377
|
+
row_factory=importlib.import_module("psycopg.rows").dict_row
|
|
378
|
+
if mapping
|
|
379
|
+
else importlib.import_module("psycopg.rows").tuple_row,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
def prepare(self):
|
|
383
|
+
pass
|
|
384
|
+
|
|
385
|
+
def sql(self, statement):
|
|
386
|
+
return statement.replace("?", "%s")
|
|
387
|
+
|
|
388
|
+
def tables(self, connection):
|
|
389
|
+
return {
|
|
390
|
+
row["tablename"] if isinstance(row, dict) else row[0]
|
|
391
|
+
for row in connection.execute(
|
|
392
|
+
"SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname=current_schema()"
|
|
393
|
+
).fetchall()
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
def indexes(self, connection):
|
|
397
|
+
return {
|
|
398
|
+
row["indexname"] if isinstance(row, dict) else row[0]
|
|
399
|
+
for row in connection.execute(
|
|
400
|
+
"SELECT indexname FROM pg_catalog.pg_indexes WHERE schemaname=current_schema()"
|
|
401
|
+
).fetchall()
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
def schema_lock(self, connection):
|
|
405
|
+
self.execute(
|
|
406
|
+
connection,
|
|
407
|
+
"SELECT pg_advisory_xact_lock(?)",
|
|
408
|
+
(self.lock_key("dockflow-core", "schema-registry-v1"),),
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
def transaction_lock(self, connection, key, *, shared=False):
|
|
412
|
+
function = "pg_advisory_xact_lock_shared" if shared else "pg_advisory_xact_lock"
|
|
413
|
+
self.execute(connection, f"SELECT {function}(?)", (key,))
|
|
414
|
+
|
|
415
|
+
def try_lock(self, connection, key, *, shared=False):
|
|
416
|
+
function = "pg_try_advisory_lock_shared" if shared else "pg_try_advisory_lock"
|
|
417
|
+
return self.execute(connection, f"SELECT {function}(?)", (key,)).fetchone()[0]
|
|
418
|
+
|
|
419
|
+
def unlock(self, connection, key, *, shared=False):
|
|
420
|
+
function = "pg_advisory_unlock_shared" if shared else "pg_advisory_unlock"
|
|
421
|
+
self.execute(connection, f"SELECT {function}(?)", (key,))
|
|
422
|
+
|
|
423
|
+
def owner_connection(self, lock_id, *, exclusive=False):
|
|
424
|
+
connection = self.connect()
|
|
425
|
+
try:
|
|
426
|
+
function = (
|
|
427
|
+
"pg_try_advisory_lock" if exclusive else "pg_try_advisory_lock_shared"
|
|
428
|
+
)
|
|
429
|
+
if not self.execute(
|
|
430
|
+
connection, f"SELECT {function}(?)", (lock_id,)
|
|
431
|
+
).fetchone()[0]:
|
|
432
|
+
raise OwnerConflict
|
|
433
|
+
return connection, None
|
|
434
|
+
except BaseException:
|
|
435
|
+
connection.close()
|
|
436
|
+
raise
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def database(options: dict) -> Database:
|
|
440
|
+
backends = {"sqlite": SQLiteDatabase, "postgresql": PostgreSQLDatabase}
|
|
441
|
+
try:
|
|
442
|
+
cls = backends[options["backend"]]
|
|
443
|
+
except (KeyError, TypeError):
|
|
444
|
+
raise DockflowError("Unsupported database backend") from None
|
|
445
|
+
return cls(options)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def reject_network_sqlite(path: Path) -> None:
|
|
449
|
+
"""Reject known Linux network mounts; other platforms require operator care."""
|
|
450
|
+
mounts = Path("/proc/mounts")
|
|
451
|
+
if not mounts.exists():
|
|
452
|
+
return
|
|
453
|
+
try:
|
|
454
|
+
rows = [line.split() for line in mounts.read_text().splitlines()]
|
|
455
|
+
except OSError:
|
|
456
|
+
return
|
|
457
|
+
matching = [
|
|
458
|
+
(Path(row[1].replace("\\040", " ")), row[2])
|
|
459
|
+
for row in rows
|
|
460
|
+
if len(row) >= 3 and path.is_relative_to(Path(row[1].replace("\\040", " ")))
|
|
461
|
+
]
|
|
462
|
+
if matching:
|
|
463
|
+
_, filesystem = max(matching, key=lambda row: len(row[0].parts))
|
|
464
|
+
if filesystem in {
|
|
465
|
+
"nfs",
|
|
466
|
+
"nfs4",
|
|
467
|
+
"cifs",
|
|
468
|
+
"smbfs",
|
|
469
|
+
"9p",
|
|
470
|
+
"ceph",
|
|
471
|
+
"afs",
|
|
472
|
+
"fuse.sshfs",
|
|
473
|
+
"fuse.s3fs",
|
|
474
|
+
"fuse.rclone",
|
|
475
|
+
"glusterfs",
|
|
476
|
+
"lustre",
|
|
477
|
+
}:
|
|
478
|
+
raise DockflowError("SQLite requires a local filesystem")
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def configured_database(options: dict | Database) -> Database:
|
|
482
|
+
if isinstance(options, Database):
|
|
483
|
+
return options
|
|
484
|
+
if not isinstance(options, dict):
|
|
485
|
+
raise DockflowError("Database configuration must be an object")
|
|
486
|
+
backend = options.get("backend")
|
|
487
|
+
field = "database" if backend == "sqlite" else "dsn_env"
|
|
488
|
+
if (
|
|
489
|
+
not isinstance(backend, str)
|
|
490
|
+
or backend not in {"sqlite", "postgresql"}
|
|
491
|
+
or set(options) != {"backend", field}
|
|
492
|
+
or not isinstance(options.get(field), str)
|
|
493
|
+
or not options[field]
|
|
494
|
+
):
|
|
495
|
+
raise DockflowError("Expected SQLite database path or PostgreSQL dsn_env")
|
|
496
|
+
if backend == "sqlite":
|
|
497
|
+
if not Path(options[field]).is_absolute():
|
|
498
|
+
raise DockflowError("Database path must be absolute")
|
|
499
|
+
elif not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", options[field]):
|
|
500
|
+
raise DockflowError("Database dsn_env must name an environment variable")
|
|
501
|
+
return database(options)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Errors shared by the core and interface adapters."""
|
|
2
|
+
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DockflowError(Exception):
|
|
7
|
+
def __init__(
|
|
8
|
+
self,
|
|
9
|
+
message: str,
|
|
10
|
+
code: str = "INVALID_ARGUMENT",
|
|
11
|
+
status: int = 400,
|
|
12
|
+
details: list[dict] | None = None,
|
|
13
|
+
):
|
|
14
|
+
super().__init__(message)
|
|
15
|
+
self.message = message
|
|
16
|
+
self.code = code
|
|
17
|
+
self.status = status
|
|
18
|
+
self.details = details or []
|
|
19
|
+
self.request_id = str(uuid.uuid4())
|
|
20
|
+
|
|
21
|
+
def as_dict(self) -> dict:
|
|
22
|
+
return {
|
|
23
|
+
"code": self.code,
|
|
24
|
+
"message": self.message,
|
|
25
|
+
"details": self.details,
|
|
26
|
+
"request_id": self.request_id,
|
|
27
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Content and durability primitives with no storage or domain policy."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import uuid
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import BinaryIO
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def reject_symlinks(path: Path) -> None:
|
|
11
|
+
path = path.absolute()
|
|
12
|
+
for candidate in (path, *path.parents):
|
|
13
|
+
if candidate.is_symlink():
|
|
14
|
+
raise ValueError("Paths cannot contain symlinks")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def hash_stream(stream: BinaryIO) -> tuple[int, str]:
|
|
18
|
+
count = 0
|
|
19
|
+
digest = hashlib.sha256()
|
|
20
|
+
while chunk := stream.read(1024 * 1024):
|
|
21
|
+
count += len(chunk)
|
|
22
|
+
digest.update(chunk)
|
|
23
|
+
return count, digest.hexdigest()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def fsync_directory(path: Path) -> None:
|
|
27
|
+
if os.name != "nt":
|
|
28
|
+
descriptor = os.open(
|
|
29
|
+
path,
|
|
30
|
+
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
31
|
+
)
|
|
32
|
+
try:
|
|
33
|
+
os.fsync(descriptor)
|
|
34
|
+
finally:
|
|
35
|
+
os.close(descriptor)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def fsync_ancestors(path: Path) -> None:
|
|
39
|
+
"""Flush every potentially new directory entry, starting at the leaf."""
|
|
40
|
+
for directory in (path, *path.parents):
|
|
41
|
+
fsync_directory(directory)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def atomic_write(path: Path, data: bytes) -> None:
|
|
45
|
+
"""Replace a file and durably acknowledge its new directory entry."""
|
|
46
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
48
|
+
try:
|
|
49
|
+
with temporary.open("xb") as stream:
|
|
50
|
+
stream.write(data)
|
|
51
|
+
stream.flush()
|
|
52
|
+
os.fsync(stream.fileno())
|
|
53
|
+
os.replace(temporary, path)
|
|
54
|
+
fsync_directory(path.parent)
|
|
55
|
+
finally:
|
|
56
|
+
temporary.unlink(missing_ok=True)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Portable local database file ownership and path validation."""
|
|
2
|
+
|
|
3
|
+
import errno
|
|
4
|
+
import os
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def owner_lock(stream: Any, *, acquire: bool) -> None:
|
|
9
|
+
try:
|
|
10
|
+
if os.name == "nt":
|
|
11
|
+
import msvcrt
|
|
12
|
+
|
|
13
|
+
# A byte-range lock may extend beyond EOF. Never write before exclusion.
|
|
14
|
+
stream.seek(0)
|
|
15
|
+
msvcrt.locking(
|
|
16
|
+
stream.fileno(), msvcrt.LK_NBLCK if acquire else msvcrt.LK_UNLCK, 1
|
|
17
|
+
)
|
|
18
|
+
else:
|
|
19
|
+
import fcntl
|
|
20
|
+
|
|
21
|
+
fcntl.flock(
|
|
22
|
+
stream.fileno(),
|
|
23
|
+
fcntl.LOCK_EX | fcntl.LOCK_NB if acquire else fcntl.LOCK_UN,
|
|
24
|
+
)
|
|
25
|
+
except ImportError as exc:
|
|
26
|
+
raise OSError(errno.ENOSYS, "OS locking backend unavailable") from exc
|
|
File without changes
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Consumer-supplied schema descriptions, without a registry of consumers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from .database import Database
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class Schema:
|
|
16
|
+
name: str
|
|
17
|
+
version: int
|
|
18
|
+
statements: tuple[str, ...]
|
|
19
|
+
tables: frozenset[str]
|
|
20
|
+
indexes: frozenset[str] = frozenset()
|
|
21
|
+
initialize: Callable[[Database, Any], None] | None = None
|
|
22
|
+
|
|
23
|
+
def __post_init__(self):
|
|
24
|
+
if (
|
|
25
|
+
not re.fullmatch(r"[a-z][a-z0-9_-]*", self.name)
|
|
26
|
+
or type(self.version) is not int
|
|
27
|
+
or self.version < 1
|
|
28
|
+
or not self.tables
|
|
29
|
+
or any(
|
|
30
|
+
not re.fullmatch(r"[a-z_][a-z0-9_]*", name)
|
|
31
|
+
for name in self.tables | self.indexes
|
|
32
|
+
)
|
|
33
|
+
or "dockflow_schema_components" in self.tables
|
|
34
|
+
):
|
|
35
|
+
raise ValueError("Invalid schema definition")
|