python-corekit 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corekit/connections/sql/__init__.py +2 -1
- corekit/connections/sql/connection.py +15 -15
- corekit/connections/sql/operations/base.py +9 -9
- corekit/connections/sql/operations/statements.py +12 -11
- corekit/connections/sql/table.py +6 -3
- corekit/etl/__init__.py +4 -16
- corekit/integrations/__init__.py +30 -0
- corekit/integrations/adapter.py +72 -0
- corekit/integrations/loader.py +41 -0
- corekit/integrations/normalizer.py +28 -0
- corekit/integrations/pipeline.py +136 -0
- corekit/integrations/schemas.py +49 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.3.dist-info}/METADATA +1 -1
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.3.dist-info}/RECORD +17 -11
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.3.dist-info}/WHEEL +0 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.3.dist-info}/licenses/LICENSE +0 -0
- {python_corekit-0.6.2.dist-info → python_corekit-0.6.3.dist-info}/top_level.txt +0 -0
|
@@ -16,7 +16,7 @@ from corekit.connections.sql.operations import (
|
|
|
16
16
|
Update,
|
|
17
17
|
)
|
|
18
18
|
from corekit.connections.sql.query import Query
|
|
19
|
-
from corekit.connections.sql.table import NamedTable, TableExportItem
|
|
19
|
+
from corekit.connections.sql.table import NamedTable, TableExportItem, TTable
|
|
20
20
|
|
|
21
21
|
__all__ = [
|
|
22
22
|
"Conditional",
|
|
@@ -35,5 +35,6 @@ __all__ = [
|
|
|
35
35
|
"Select",
|
|
36
36
|
"SQLConnection",
|
|
37
37
|
"TableExportItem",
|
|
38
|
+
"TTable",
|
|
38
39
|
"Update",
|
|
39
40
|
]
|
|
@@ -26,7 +26,7 @@ from corekit.config import get_settings
|
|
|
26
26
|
from corekit.connections import Connectable
|
|
27
27
|
from corekit.connections.sql.operations import DqlOperation, Operation, Select
|
|
28
28
|
from corekit.connections.sql.query import Query
|
|
29
|
-
from corekit.connections.sql.table import
|
|
29
|
+
from corekit.connections.sql.table import TTable
|
|
30
30
|
|
|
31
31
|
__all__ = ["PaginatedResult", "SQLConnection"]
|
|
32
32
|
|
|
@@ -261,7 +261,7 @@ class SQLConnection(Connectable):
|
|
|
261
261
|
self.session.merge(item)
|
|
262
262
|
self.session.commit()
|
|
263
263
|
|
|
264
|
-
def delete(self, table: type[
|
|
264
|
+
def delete(self, table: type[TTable], identifier: str) -> None:
|
|
265
265
|
"""
|
|
266
266
|
Delete a row by id.
|
|
267
267
|
"""
|
|
@@ -308,19 +308,19 @@ class SQLConnection(Connectable):
|
|
|
308
308
|
result = self.select(statement)
|
|
309
309
|
return None if result is None else result.first()
|
|
310
310
|
|
|
311
|
-
def fetch_all(self, table: type[
|
|
311
|
+
def fetch_all(self, table: type[TTable]) -> Iterator[TTable]:
|
|
312
312
|
"""
|
|
313
313
|
Every row in a table.
|
|
314
314
|
"""
|
|
315
315
|
return self._exec(select(table))
|
|
316
316
|
|
|
317
|
-
def fetch_one_by_id(self, table: type[
|
|
317
|
+
def fetch_one_by_id(self, table: type[TTable], identifier: str) -> TTable | None:
|
|
318
318
|
"""
|
|
319
319
|
A single row by id.
|
|
320
320
|
"""
|
|
321
321
|
return self.first(select(table).where(table.id == identifier))
|
|
322
322
|
|
|
323
|
-
def fetch_one_by_condition(self, table: type[
|
|
323
|
+
def fetch_one_by_condition(self, table: type[TTable], key: Any, value: Any) -> TTable | None:
|
|
324
324
|
"""
|
|
325
325
|
The first row where a column equals a value.
|
|
326
326
|
|
|
@@ -330,13 +330,13 @@ class SQLConnection(Connectable):
|
|
|
330
330
|
column = getattr(table, key) if isinstance(key, str) else key
|
|
331
331
|
return self.first(select(table).where(column == value))
|
|
332
332
|
|
|
333
|
-
def exists(self, table: type[
|
|
333
|
+
def exists(self, table: type[TTable], identifier: str) -> bool:
|
|
334
334
|
"""
|
|
335
335
|
Whether a row with this id exists.
|
|
336
336
|
"""
|
|
337
337
|
return self.fetch_one_by_id(table, identifier) is not None
|
|
338
338
|
|
|
339
|
-
def fetch_or_create_one(self, table: type[
|
|
339
|
+
def fetch_or_create_one(self, table: type[TTable], identifier: str) -> TTable:
|
|
340
340
|
"""
|
|
341
341
|
Fetch a row by id, inserting a default one if it is missing.
|
|
342
342
|
"""
|
|
@@ -346,13 +346,13 @@ class SQLConnection(Connectable):
|
|
|
346
346
|
self.insert(item)
|
|
347
347
|
return item
|
|
348
348
|
|
|
349
|
-
def fetch_many_by_query(self, query: Query) -> list[
|
|
349
|
+
def fetch_many_by_query(self, query: Query[TTable]) -> list[TTable]:
|
|
350
350
|
"""
|
|
351
351
|
Every row matching a Query.
|
|
352
352
|
"""
|
|
353
353
|
return list(self._exec(query.build()))
|
|
354
354
|
|
|
355
|
-
def delete_many_by_query(self, query: Query) -> int:
|
|
355
|
+
def delete_many_by_query(self, query: Query[TTable]) -> int:
|
|
356
356
|
"""
|
|
357
357
|
Delete every row matching a Query.
|
|
358
358
|
|
|
@@ -366,7 +366,7 @@ class SQLConnection(Connectable):
|
|
|
366
366
|
self.session.commit()
|
|
367
367
|
return len(items)
|
|
368
368
|
|
|
369
|
-
def execute(self, operation: Operation) -> Any:
|
|
369
|
+
def execute(self, operation: Operation[Any]) -> Any:
|
|
370
370
|
"""
|
|
371
371
|
Run an operation, committing if it writes.
|
|
372
372
|
|
|
@@ -380,19 +380,19 @@ class SQLConnection(Connectable):
|
|
|
380
380
|
self.session.commit()
|
|
381
381
|
return result
|
|
382
382
|
|
|
383
|
-
def fetch(self, operation: DqlOperation) -> list[
|
|
383
|
+
def fetch(self, operation: DqlOperation[TTable]) -> list[TTable]:
|
|
384
384
|
"""
|
|
385
385
|
Every row an operation selects.
|
|
386
386
|
"""
|
|
387
387
|
return list(self.execute(operation))
|
|
388
388
|
|
|
389
|
-
def fetch_one_by_query(self, query: Query) ->
|
|
389
|
+
def fetch_one_by_query(self, query: Query[TTable]) -> TTable | None:
|
|
390
390
|
"""
|
|
391
391
|
The first row matching a Query.
|
|
392
392
|
"""
|
|
393
393
|
return self.first(query.build())
|
|
394
394
|
|
|
395
|
-
def count(self, table: type[
|
|
395
|
+
def count(self, table: type[TTable]) -> int:
|
|
396
396
|
"""
|
|
397
397
|
The number of rows in a table.
|
|
398
398
|
"""
|
|
@@ -401,13 +401,13 @@ class SQLConnection(Connectable):
|
|
|
401
401
|
|
|
402
402
|
def paginate(
|
|
403
403
|
self,
|
|
404
|
-
table: type[
|
|
404
|
+
table: type[TTable],
|
|
405
405
|
offset: int = 0,
|
|
406
406
|
limit: int = 50,
|
|
407
407
|
*,
|
|
408
408
|
order_by: Any = None,
|
|
409
409
|
desc: bool = False,
|
|
410
|
-
) -> PaginatedResult:
|
|
410
|
+
) -> PaginatedResult[TTable]:
|
|
411
411
|
"""
|
|
412
412
|
A page of rows, together with the total row count.
|
|
413
413
|
|
|
@@ -3,17 +3,17 @@ Statement primitives: the shapes a database operation can take.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from abc import ABC, abstractmethod
|
|
6
|
-
from typing import Any, ClassVar
|
|
6
|
+
from typing import Any, ClassVar, Generic, Self
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel, Field
|
|
9
9
|
|
|
10
|
-
from corekit.connections.sql.table import
|
|
10
|
+
from corekit.connections.sql.table import TTable
|
|
11
11
|
from corekit.data.expressions import Expression
|
|
12
12
|
|
|
13
13
|
__all__ = ["Conditional", "DdlOperation", "DmlOperation", "DqlOperation", "Operation"]
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
class Operation(BaseModel, ABC):
|
|
16
|
+
class Operation(BaseModel, ABC, Generic[TTable]):
|
|
17
17
|
"""
|
|
18
18
|
A statement waiting to be executed.
|
|
19
19
|
|
|
@@ -28,7 +28,7 @@ class Operation(BaseModel, ABC):
|
|
|
28
28
|
#: Conditional writes (update, delete) and insert both set this; reads do not.
|
|
29
29
|
writes: ClassVar[bool] = False
|
|
30
30
|
|
|
31
|
-
table: type[
|
|
31
|
+
table: type[TTable]
|
|
32
32
|
|
|
33
33
|
@abstractmethod
|
|
34
34
|
def build(self) -> Any:
|
|
@@ -40,7 +40,7 @@ class Operation(BaseModel, ABC):
|
|
|
40
40
|
return f"{type(self).__name__}({self.table.__name__})"
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
class Conditional(Operation, ABC):
|
|
43
|
+
class Conditional(Operation[TTable], ABC, Generic[TTable]):
|
|
44
44
|
"""
|
|
45
45
|
An operation narrowed by conditions, e.g. everything but an insert.
|
|
46
46
|
|
|
@@ -51,7 +51,7 @@ class Conditional(Operation, ABC):
|
|
|
51
51
|
|
|
52
52
|
conditions: list[Any] = Field(default_factory=list)
|
|
53
53
|
|
|
54
|
-
def where(self, condition: Any) ->
|
|
54
|
+
def where(self, condition: Any) -> Self:
|
|
55
55
|
"""
|
|
56
56
|
Narrow this operation. Repeated calls combine with AND.
|
|
57
57
|
"""
|
|
@@ -75,13 +75,13 @@ class Conditional(Operation, ABC):
|
|
|
75
75
|
return statement
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
class DqlOperation(Conditional, ABC):
|
|
78
|
+
class DqlOperation(Conditional[TTable], ABC, Generic[TTable]):
|
|
79
79
|
"""
|
|
80
80
|
A read. Returns rows and changes nothing.
|
|
81
81
|
"""
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
class DmlOperation(Conditional, ABC):
|
|
84
|
+
class DmlOperation(Conditional[TTable], ABC, Generic[TTable]):
|
|
85
85
|
"""
|
|
86
86
|
A conditional write against rows: update or delete.
|
|
87
87
|
|
|
@@ -92,7 +92,7 @@ class DmlOperation(Conditional, ABC):
|
|
|
92
92
|
writes: ClassVar[bool] = True
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
class DdlOperation(Operation, ABC):
|
|
95
|
+
class DdlOperation(Operation[TTable], ABC, Generic[TTable]):
|
|
96
96
|
"""
|
|
97
97
|
A change to the schema itself.
|
|
98
98
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
The statements an application runs: select, insert, update and delete.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from typing import Any, ClassVar
|
|
5
|
+
from typing import Any, ClassVar, Generic, Self
|
|
6
6
|
|
|
7
7
|
from pydantic import Field
|
|
8
8
|
from sqlmodel import delete as _delete
|
|
@@ -12,11 +12,12 @@ from sqlmodel import select as _select
|
|
|
12
12
|
from sqlmodel import update as _update
|
|
13
13
|
|
|
14
14
|
from corekit.connections.sql.operations.base import DmlOperation, DqlOperation, Operation
|
|
15
|
+
from corekit.connections.sql.table import TTable
|
|
15
16
|
|
|
16
17
|
__all__ = ["Delete", "Insert", "Select", "Update"]
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
class Select(DqlOperation):
|
|
20
|
+
class Select(DqlOperation[TTable], Generic[TTable]):
|
|
20
21
|
"""
|
|
21
22
|
Read rows, optionally ordered, offset and capped.
|
|
22
23
|
|
|
@@ -37,7 +38,7 @@ class Select(DqlOperation):
|
|
|
37
38
|
offset_count: int = 0
|
|
38
39
|
for_update_lock: bool = False
|
|
39
40
|
|
|
40
|
-
def order_by(self, field: Any, desc: bool = False) ->
|
|
41
|
+
def order_by(self, field: Any, desc: bool = False) -> Self:
|
|
41
42
|
"""
|
|
42
43
|
Order results by a field.
|
|
43
44
|
"""
|
|
@@ -45,7 +46,7 @@ class Select(DqlOperation):
|
|
|
45
46
|
self.order_by_desc = desc
|
|
46
47
|
return self
|
|
47
48
|
|
|
48
|
-
def limit(self, limit: int) ->
|
|
49
|
+
def limit(self, limit: int) -> Self:
|
|
49
50
|
"""
|
|
50
51
|
Cap the number of rows returned.
|
|
51
52
|
|
|
@@ -54,14 +55,14 @@ class Select(DqlOperation):
|
|
|
54
55
|
self.limit_count = limit
|
|
55
56
|
return self
|
|
56
57
|
|
|
57
|
-
def offset(self, offset: int) ->
|
|
58
|
+
def offset(self, offset: int) -> Self:
|
|
58
59
|
"""
|
|
59
60
|
Skip the first ``offset`` rows.
|
|
60
61
|
"""
|
|
61
62
|
self.offset_count = offset
|
|
62
63
|
return self
|
|
63
64
|
|
|
64
|
-
def for_update(self) ->
|
|
65
|
+
def for_update(self) -> Self:
|
|
65
66
|
"""
|
|
66
67
|
Lock the matching rows until the transaction ends.
|
|
67
68
|
"""
|
|
@@ -92,7 +93,7 @@ class Select(DqlOperation):
|
|
|
92
93
|
return statement
|
|
93
94
|
|
|
94
95
|
|
|
95
|
-
class Delete(DmlOperation):
|
|
96
|
+
class Delete(DmlOperation[TTable], Generic[TTable]):
|
|
96
97
|
"""
|
|
97
98
|
Delete every row matching the conditions, in one statement.
|
|
98
99
|
|
|
@@ -114,7 +115,7 @@ class Delete(DmlOperation):
|
|
|
114
115
|
return self._apply_conditions(_delete(self.table))
|
|
115
116
|
|
|
116
117
|
|
|
117
|
-
class Update(DmlOperation):
|
|
118
|
+
class Update(DmlOperation[TTable], Generic[TTable]):
|
|
118
119
|
"""
|
|
119
120
|
Set columns on every row matching the conditions, in one statement.
|
|
120
121
|
|
|
@@ -125,7 +126,7 @@ class Update(DmlOperation):
|
|
|
125
126
|
|
|
126
127
|
values: dict[str, Any] = Field(default_factory=dict)
|
|
127
128
|
|
|
128
|
-
def set(self, **values: Any) ->
|
|
129
|
+
def set(self, **values: Any) -> Self:
|
|
129
130
|
"""
|
|
130
131
|
Columns to write, as keyword arguments.
|
|
131
132
|
"""
|
|
@@ -146,7 +147,7 @@ class Update(DmlOperation):
|
|
|
146
147
|
return self._apply_conditions(_update(self.table)).values(**self.values)
|
|
147
148
|
|
|
148
149
|
|
|
149
|
-
class Insert(Operation):
|
|
150
|
+
class Insert(Operation[TTable], Generic[TTable]):
|
|
150
151
|
"""
|
|
151
152
|
Add rows. A write, so ``SQLConnection.execute`` commits it.
|
|
152
153
|
|
|
@@ -163,7 +164,7 @@ class Insert(Operation):
|
|
|
163
164
|
|
|
164
165
|
rows: list[dict[str, Any]] = Field(default_factory=list)
|
|
165
166
|
|
|
166
|
-
def add(self, **values: Any) ->
|
|
167
|
+
def add(self, **values: Any) -> Self:
|
|
167
168
|
"""
|
|
168
169
|
Append one row.
|
|
169
170
|
"""
|
corekit/connections/sql/table.py
CHANGED
|
@@ -3,14 +3,14 @@ Base table type.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
|
-
from typing import Any, ClassVar
|
|
6
|
+
from typing import Any, ClassVar, Self, TypeVar
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel
|
|
9
9
|
from sqlmodel import SQLModel
|
|
10
10
|
|
|
11
11
|
from corekit.registry import SmartRegistry
|
|
12
12
|
|
|
13
|
-
__all__ = ["NamedTable", "TableExportItem"]
|
|
13
|
+
__all__ = ["NamedTable", "TableExportItem", "TTable"]
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class TableExportItem(BaseModel):
|
|
@@ -85,7 +85,7 @@ class NamedTable(SQLModel):
|
|
|
85
85
|
return name or cls.__name__
|
|
86
86
|
|
|
87
87
|
@classmethod
|
|
88
|
-
def create_default(cls, identifier: str, **kwargs: Any) ->
|
|
88
|
+
def create_default(cls, identifier: str, **kwargs: Any) -> Self:
|
|
89
89
|
"""
|
|
90
90
|
Build an instance carrying only its identifier and any supplied fields.
|
|
91
91
|
"""
|
|
@@ -98,3 +98,6 @@ class NamedTable(SQLModel):
|
|
|
98
98
|
This row as a portable record, tagged with its database table name.
|
|
99
99
|
"""
|
|
100
100
|
return TableExportItem(table_name=self.table_name, data=self.model_dump())
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
TTable = TypeVar("TTable", bound=NamedTable)
|
corekit/etl/__init__.py
CHANGED
|
@@ -1,21 +1,9 @@
|
|
|
1
1
|
"""
|
|
2
|
-
ETL pipelines.
|
|
2
|
+
ETL pipelines (legacy).
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class UserPipeline(BaseETLOrchestrator):
|
|
8
|
-
'''
|
|
9
|
-
Loads users from the API into the warehouse.
|
|
10
|
-
'''
|
|
11
|
-
extractor = UserExtractor
|
|
12
|
-
transformer = UserTransformer
|
|
13
|
-
loader = UserLoader
|
|
14
|
-
|
|
15
|
-
await UserPipeline().run()
|
|
16
|
-
|
|
17
|
-
The run loop streams, so memory use is bounded by the batch size rather than by
|
|
18
|
-
how much data the source holds.
|
|
4
|
+
Prefer ``corekit.integrations`` for new work: ``BaseAdapter`` / ``BaseNormalizer``
|
|
5
|
+
/ ``BaseLoader`` / ``BasePipeline``. This package remains for existing tests and
|
|
6
|
+
imports; new vendors should not extend Extractor/Transformer naming.
|
|
19
7
|
"""
|
|
20
8
|
|
|
21
9
|
from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Integration framework: clients, adapters, normalizers, loaders, pipelines.
|
|
3
|
+
|
|
4
|
+
Layering::
|
|
5
|
+
|
|
6
|
+
BaseApiClient — wire + auth to the external system
|
|
7
|
+
BaseNormalizer — raw payload → concrete BaseModel (never dict-out)
|
|
8
|
+
BaseAdapter — named ops (interactive) + optional bulk sources (sync)
|
|
9
|
+
BaseLoader — persist normalized items into the host system
|
|
10
|
+
BasePipeline — stream adapter bulk → loader (nightly / full sync)
|
|
11
|
+
|
|
12
|
+
Interactive Integrations call ``adapter.run(...)``. Catalog syncs call
|
|
13
|
+
``pipeline.run()``. Do not route point lookups through the pipeline.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from corekit.integrations.adapter import BaseAdapter
|
|
17
|
+
from corekit.integrations.loader import BaseLoader
|
|
18
|
+
from corekit.integrations.normalizer import BaseNormalizer
|
|
19
|
+
from corekit.integrations.pipeline import BasePipeline
|
|
20
|
+
from corekit.integrations.schemas import LoadableBatch, NormalizedItem, OperationClass
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BaseAdapter",
|
|
24
|
+
"BaseLoader",
|
|
25
|
+
"BaseNormalizer",
|
|
26
|
+
"BasePipeline",
|
|
27
|
+
"LoadableBatch",
|
|
28
|
+
"NormalizedItem",
|
|
29
|
+
"OperationClass",
|
|
30
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Vendor adapter: named operations plus optional bulk sources for sync pipelines.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from collections.abc import AsyncIterable, Callable
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
from corekit.http import BadRequestException, BaseApiClient
|
|
14
|
+
from corekit.integrations.normalizer import BaseNormalizer
|
|
15
|
+
from corekit.integrations.schemas import NormalizedItem, OperationClass
|
|
16
|
+
from corekit.observability import Loggable
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BaseAdapter(Loggable, ABC):
|
|
20
|
+
"""
|
|
21
|
+
Integration surface for one external system.
|
|
22
|
+
|
|
23
|
+
- **Named ops** (``run``): interactive request/response (search, create, …).
|
|
24
|
+
- **Bulk sources** (``iter_bulk``): streaming catalog/sync pulls for pipelines.
|
|
25
|
+
|
|
26
|
+
Always constructed with a ``BaseApiClient`` and a ``BaseNormalizer``. Ops and
|
|
27
|
+
bulk sources return ``BaseModel`` instances — never raw dicts.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, client: BaseApiClient, normalizer: BaseNormalizer[Any]) -> None:
|
|
31
|
+
super().__init__()
|
|
32
|
+
self.client = client
|
|
33
|
+
self.normalizer = normalizer
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def operation_classes(self) -> dict[str, OperationClass]:
|
|
37
|
+
"""
|
|
38
|
+
Map operation key → read/write class.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def operations(self) -> frozenset[str]:
|
|
43
|
+
return frozenset(self.operation_classes())
|
|
44
|
+
|
|
45
|
+
def operation_class(self, operation: str) -> OperationClass | None:
|
|
46
|
+
return self.operation_classes().get(operation)
|
|
47
|
+
|
|
48
|
+
def require_operation(self, operation: str) -> OperationClass:
|
|
49
|
+
op_class = self.operation_class(operation)
|
|
50
|
+
if op_class is None:
|
|
51
|
+
raise BadRequestException(detail=f"Unknown operation '{operation}'")
|
|
52
|
+
return op_class
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
async def run(self, operation: str, payload: dict[str, Any], **context: Any) -> BaseModel:
|
|
56
|
+
"""
|
|
57
|
+
Execute a named operation. Return a normalized ``BaseModel`` (never secrets).
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def bulk_sources(self) -> list[Callable[..., AsyncIterable[NormalizedItem]]]:
|
|
61
|
+
"""
|
|
62
|
+
Optional streaming sources for full sync. Default: none.
|
|
63
|
+
"""
|
|
64
|
+
return []
|
|
65
|
+
|
|
66
|
+
async def iter_bulk(self, *args: Any, **kwargs: Any) -> AsyncIterable[NormalizedItem]:
|
|
67
|
+
"""
|
|
68
|
+
Yield normalized items from every bulk source.
|
|
69
|
+
"""
|
|
70
|
+
for source in self.bulk_sources():
|
|
71
|
+
async for item in source(*args, **kwargs):
|
|
72
|
+
yield item
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Persist normalized items into the host system.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
|
|
9
|
+
from corekit.integrations.schemas import LoadableBatch, NormalizedItem
|
|
10
|
+
from corekit.observability import Loggable
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseLoader(Loggable, ABC):
|
|
14
|
+
"""
|
|
15
|
+
Batches normalized items and writes them to a destination (DB, namespace, …).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
super().__init__()
|
|
20
|
+
self.batch: LoadableBatch = LoadableBatch()
|
|
21
|
+
|
|
22
|
+
def _should_load(self, min_ops: int) -> bool:
|
|
23
|
+
return bool(len(self.batch)) and len(self.batch) >= min_ops
|
|
24
|
+
|
|
25
|
+
def add_item(self, item: NormalizedItem) -> None:
|
|
26
|
+
self.batch.add_item(item)
|
|
27
|
+
|
|
28
|
+
async def flush(self) -> None:
|
|
29
|
+
await self.load(min_ops=0)
|
|
30
|
+
|
|
31
|
+
async def load(self, min_ops: int) -> None:
|
|
32
|
+
if not self._should_load(min_ops):
|
|
33
|
+
return
|
|
34
|
+
await self._load(self.batch)
|
|
35
|
+
self.batch.clear()
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def _load(self, batch: LoadableBatch) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Write a batch to its destination.
|
|
41
|
+
"""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Translate raw external payloads into known Pydantic models.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from typing import Any, Generic, TypeVar
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
|
|
12
|
+
from corekit.observability import Loggable
|
|
13
|
+
|
|
14
|
+
TModel = TypeVar("TModel", bound=BaseModel)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BaseNormalizer(Loggable, ABC, Generic[TModel]):
|
|
18
|
+
"""
|
|
19
|
+
Pure translation from vendor/raw payloads to a concrete ``BaseModel``.
|
|
20
|
+
|
|
21
|
+
Output is always structured. Dict-in/dict-out is not a normalizer.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def normalize(self, raw: Any) -> TModel:
|
|
26
|
+
"""
|
|
27
|
+
Return a typed model downstream code can rely on.
|
|
28
|
+
"""
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bulk sync: stream adapter bulk sources into a loader.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, ClassVar
|
|
9
|
+
|
|
10
|
+
from corekit.integrations.adapter import BaseAdapter
|
|
11
|
+
from corekit.integrations.loader import BaseLoader
|
|
12
|
+
from corekit.observability import Loggable
|
|
13
|
+
from corekit.registry import SmartRegistry
|
|
14
|
+
|
|
15
|
+
__all__ = ["BasePipeline"]
|
|
16
|
+
|
|
17
|
+
DEFAULT_BATCH_SIZE = 100
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class PipelineStage:
|
|
22
|
+
"""
|
|
23
|
+
One declared stage on a pipeline class.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
attribute: str
|
|
27
|
+
base_class: type
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def label(self) -> str:
|
|
31
|
+
return self.base_class.__name__.removeprefix("Base")
|
|
32
|
+
|
|
33
|
+
def declared_on(self, cls: type) -> Any:
|
|
34
|
+
return cls.__dict__.get(self.attribute)
|
|
35
|
+
|
|
36
|
+
def validate(self, cls: type) -> None:
|
|
37
|
+
stage = getattr(cls, self.attribute, None)
|
|
38
|
+
if stage is None:
|
|
39
|
+
raise TypeError(f"{cls.__name__} is missing its {self.attribute}. Set {self.attribute} = Your{self.label}.")
|
|
40
|
+
if not (isinstance(stage, type) and issubclass(stage, self.base_class)):
|
|
41
|
+
raise TypeError(
|
|
42
|
+
f"{cls.__name__}.{self.attribute} must be a {self.base_class.__name__} subclass, "
|
|
43
|
+
f"got {stage!r}. Assign the class itself, not an instance."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class BasePipeline(Loggable):
|
|
48
|
+
"""
|
|
49
|
+
Runs adapter bulk sync into a loader.
|
|
50
|
+
|
|
51
|
+
Interactive named ops stay on ``BaseAdapter.run`` — this class is for
|
|
52
|
+
nightly/full pulls only.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
adapter: ClassVar[type[BaseAdapter] | None] = None
|
|
56
|
+
loader: ClassVar[type[BaseLoader] | None] = None
|
|
57
|
+
batch_size: ClassVar[int] = DEFAULT_BATCH_SIZE
|
|
58
|
+
|
|
59
|
+
__registry__: SmartRegistry = SmartRegistry()
|
|
60
|
+
__binding_only__: ClassVar[bool] = False
|
|
61
|
+
|
|
62
|
+
STAGES: ClassVar[list[PipelineStage]] = [
|
|
63
|
+
PipelineStage(attribute="adapter", base_class=BaseAdapter),
|
|
64
|
+
PipelineStage(attribute="loader", base_class=BaseLoader),
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
68
|
+
super().__init_subclass__(**kwargs)
|
|
69
|
+
if cls.__dict__.get("__binding_only__"):
|
|
70
|
+
return
|
|
71
|
+
if not any(stage.declared_on(cls) for stage in cls.STAGES):
|
|
72
|
+
return
|
|
73
|
+
for stage in cls.STAGES:
|
|
74
|
+
stage.validate(cls)
|
|
75
|
+
BasePipeline.__registry__[cls.__name__] = cls
|
|
76
|
+
|
|
77
|
+
def __class_getitem__(cls, item: tuple[type, ...]) -> Any:
|
|
78
|
+
if not isinstance(item, tuple) or len(item) != 2:
|
|
79
|
+
raise TypeError(f"{cls.__name__}[...] takes exactly two stages: {cls.__name__}[Adapter, Loader]")
|
|
80
|
+
bound = {stage.attribute: value for stage, value in zip(cls.STAGES, item, strict=True)}
|
|
81
|
+
return type(f"{cls.__name__}_{item[0].__name__}", (cls,), bound)
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def get_pipeline_types(cls) -> SmartRegistry:
|
|
85
|
+
return cls.__registry__
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def get_pipeline_by_name(cls, name: str) -> type[BasePipeline] | None:
|
|
89
|
+
return cls.__registry__.get(name)
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
adapter: BaseAdapter | None = None,
|
|
94
|
+
loader: BaseLoader | None = None,
|
|
95
|
+
batch_size: int | None = None,
|
|
96
|
+
on_item_error: str = "raise",
|
|
97
|
+
) -> None:
|
|
98
|
+
if on_item_error not in ("raise", "skip"):
|
|
99
|
+
raise ValueError("on_item_error must be 'raise' or 'skip'")
|
|
100
|
+
super().__init__()
|
|
101
|
+
self.batch_size = batch_size if batch_size is not None else type(self).batch_size
|
|
102
|
+
self.on_item_error = on_item_error
|
|
103
|
+
self._adapter = adapter
|
|
104
|
+
self._loader = loader
|
|
105
|
+
|
|
106
|
+
def _build(self, override: Any, name: str) -> Any:
|
|
107
|
+
if override is not None:
|
|
108
|
+
return override
|
|
109
|
+
stage_class = getattr(type(self), name, None)
|
|
110
|
+
if stage_class is None:
|
|
111
|
+
raise TypeError(f"{type(self).__name__} has no {name}. Declare one, or pass {name}=...")
|
|
112
|
+
return stage_class()
|
|
113
|
+
|
|
114
|
+
async def run(self) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Stream ``adapter.iter_bulk()`` into the loader until exhausted.
|
|
117
|
+
"""
|
|
118
|
+
adapter = self._build(self._adapter, "adapter")
|
|
119
|
+
loader = self._build(self._loader, "loader")
|
|
120
|
+
|
|
121
|
+
count = 0
|
|
122
|
+
skipped = 0
|
|
123
|
+
async for item in adapter.iter_bulk():
|
|
124
|
+
try:
|
|
125
|
+
loader.add_item(item)
|
|
126
|
+
except Exception:
|
|
127
|
+
if self.on_item_error == "raise":
|
|
128
|
+
raise
|
|
129
|
+
skipped += 1
|
|
130
|
+
self.exception(f"{type(self).__name__} skipped an item")
|
|
131
|
+
continue
|
|
132
|
+
await loader.load(min_ops=self.batch_size)
|
|
133
|
+
count += 1
|
|
134
|
+
|
|
135
|
+
await loader.flush()
|
|
136
|
+
self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared types for the integration framework.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import Iterator
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, Field
|
|
10
|
+
|
|
11
|
+
from corekit.schemas import StringEnum
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class OperationClass(StringEnum):
|
|
15
|
+
"""
|
|
16
|
+
Whether a named adapter operation reads from or writes to the external system.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
READ = "read"
|
|
20
|
+
WRITE = "write"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NormalizedItem(BaseModel):
|
|
24
|
+
"""
|
|
25
|
+
One unit of already-normalized data ready for a loader or API response.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
data: BaseModel
|
|
29
|
+
source: str | None = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class LoadableBatch(BaseModel):
|
|
33
|
+
"""
|
|
34
|
+
Buffered normalized items awaiting a loader flush.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
items: list[NormalizedItem] = Field(default_factory=list)
|
|
38
|
+
|
|
39
|
+
def __iter__(self) -> Iterator[NormalizedItem]:
|
|
40
|
+
yield from self.items
|
|
41
|
+
|
|
42
|
+
def __len__(self) -> int:
|
|
43
|
+
return len(self.items)
|
|
44
|
+
|
|
45
|
+
def add_item(self, item: NormalizedItem) -> None:
|
|
46
|
+
self.items.append(item)
|
|
47
|
+
|
|
48
|
+
def clear(self) -> None:
|
|
49
|
+
self.items.clear()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-corekit
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.3
|
|
4
4
|
Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, LLM chat/tool primitives, and HTML scraping
|
|
5
5
|
Author: Steven Jacobsen
|
|
6
6
|
License-Expression: MIT
|
|
@@ -21,10 +21,10 @@ corekit/connections/decorators.py,sha256=7xfmLuMr3kkceEkkxHxY_Pen2OakJEVukXtiOYA
|
|
|
21
21
|
corekit/connections/registry.py,sha256=BR04dGthcGCtON2MxNsl-TlhRgbRlvGj0RLymDFBkGQ,3119
|
|
22
22
|
corekit/connections/redis/__init__.py,sha256=BsWLZNqpKAqIoreHGLEtftnMqVw-ESK8jbexDAughag,174
|
|
23
23
|
corekit/connections/redis/connection.py,sha256=H6r5WsDfXFgIQlp6vq3rKjp-oBFQoE5h_YaJXZEvHIs,9850
|
|
24
|
-
corekit/connections/sql/__init__.py,sha256=
|
|
25
|
-
corekit/connections/sql/connection.py,sha256=
|
|
24
|
+
corekit/connections/sql/__init__.py,sha256=hZxBBk8coC2t2D2sBm_sSxZrqgVpX1Q3C6JN3muNWMc,904
|
|
25
|
+
corekit/connections/sql/connection.py,sha256=SbE9GQhGcaf0-74LSDSzyh4xyQFlmRzLKPsix2d_eMQ,12959
|
|
26
26
|
corekit/connections/sql/query.py,sha256=8s5atIssmN2l9FCL2IHnIQqwxOk0fA_mPNh36koLhoE,233
|
|
27
|
-
corekit/connections/sql/table.py,sha256=
|
|
27
|
+
corekit/connections/sql/table.py,sha256=7nkSw6S9el_htajHsPeG8Mj6oCuasZERfc3f_kIOchg,3189
|
|
28
28
|
corekit/connections/sql/fields/__init__.py,sha256=2I-TCsvi321yQhesKNZgY1T77FtATZ6o7pw0MNHtbAo,184
|
|
29
29
|
corekit/connections/sql/fields/jsonb.py,sha256=wGjtCP6v3i2bX73X-VVqBNuDNr0-MeWIosc6nvWyO9Q,2313
|
|
30
30
|
corekit/connections/sql/migration/__init__.py,sha256=NBQyKdz84y6Y_QJVTJQVfHz0WrY1CXNoeko113GUoTg,1652
|
|
@@ -33,8 +33,8 @@ corekit/connections/sql/migration/operations.py,sha256=UIO0TPJceo0vtGsNQiATn7i7s
|
|
|
33
33
|
corekit/connections/sql/migration/registry.py,sha256=M1iS7v7aKYfgyf_grvF-6Ogn4KfQ1EEWbA1MO02CiB0,6728
|
|
34
34
|
corekit/connections/sql/migration/table.py,sha256=CpOkJl-4rcG2hTZYq-Yt7OvGTpVhVJ42IrubVO3BUvs,666
|
|
35
35
|
corekit/connections/sql/operations/__init__.py,sha256=f6fnAbD3k8ALXwcwJ31fIHQwkvud_WqmUod_apTErpY,480
|
|
36
|
-
corekit/connections/sql/operations/base.py,sha256=
|
|
37
|
-
corekit/connections/sql/operations/statements.py,sha256=
|
|
36
|
+
corekit/connections/sql/operations/base.py,sha256=T5HY-0K_FQuPqmtSZZG5neVZQO5L4xRrEoErFu_9MZA,3401
|
|
37
|
+
corekit/connections/sql/operations/statements.py,sha256=S8i_687rCZV7Aveni0DECqWT5nn7mpl5yEKmADKSHbw,5397
|
|
38
38
|
corekit/crypto/__init__.py,sha256=Qf_FCUoG_jb7Q9tnN4ctX8I6xyaeaJwk74ag_bSchAQ,87
|
|
39
39
|
corekit/crypto/constants.py,sha256=m-uq7_EJ4XfUTSZxFv6h429UkKSrGTj3o_9DfRFGyvU,298
|
|
40
40
|
corekit/crypto/enum.py,sha256=0csj06cATCYDPY_QTNy66j_hB16WCqcHg8tJOO0lMNU,212
|
|
@@ -59,7 +59,7 @@ corekit/docker/disk/__init__.py,sha256=ZTr77TJuUdVvv6wbWXvYW0I5oVz45EwzbGjbjrUy1
|
|
|
59
59
|
corekit/docker/disk/constants.py,sha256=4fMEAdWwHbiBbe7aj3vV4molxRGZ3eLl1tyIPQ47D1o,465
|
|
60
60
|
corekit/docker/disk/inspector.py,sha256=YBRCX6upUclOPyu2PT2VkfyXKUPXJUv6Wwd_XYvA7lU,8156
|
|
61
61
|
corekit/docker/disk/models.py,sha256=9z8dTUyTEp8NfMmzBul2yp8sLAcEjoMytz_gCj7veKY,733
|
|
62
|
-
corekit/etl/__init__.py,sha256=
|
|
62
|
+
corekit/etl/__init__.py,sha256=4vN5ZccUjaYsunANSeV7fsl0P1dVhHi1u_iP4WZ4v9M,1152
|
|
63
63
|
corekit/etl/connection.py,sha256=_gWqR36x5f7qb5IYJjrc_7d_6oU2CXm1QeSNCCh0coo,1270
|
|
64
64
|
corekit/etl/orchestrator.py,sha256=GkecTu8gKt17ePDM9PJBUuZTPYoPMdK1yYzBHjMdw3A,7959
|
|
65
65
|
corekit/etl/schemas.py,sha256=3z0qZBintCO45BcxT6TY0Boi_ibrRa0z85wBJvLXNd4,385
|
|
@@ -116,6 +116,12 @@ corekit/http/exponential_backoff.py,sha256=5jjGTspX_BC9aBKPr5Y5dNhJqXesmljsjJoQ6
|
|
|
116
116
|
corekit/http/response.py,sha256=t_nGIwA-uyuQlJBGMgxbodCNjRnqc1xUcxMsbEXeMBM,3415
|
|
117
117
|
corekit/http/status.py,sha256=rZ-cdliG9EAvWMTzvxjqjipRkZRc6RgSzcBY4Kewdg0,2377
|
|
118
118
|
corekit/http/stream.py,sha256=lkERyCXz5amOWzoE-b12iftf3l_G-ghwSvN1lL5lUio,3257
|
|
119
|
+
corekit/integrations/__init__.py,sha256=Y9I6-A96pO8St-Gj30O1meOH-LVvc1MPPz2Wp0-5QgM,1063
|
|
120
|
+
corekit/integrations/adapter.py,sha256=9KOZpDHWQdPXCU01_d7e1L8bPqnrSiFFKf5DJBdzazc,2464
|
|
121
|
+
corekit/integrations/loader.py,sha256=iiSdR3dx7HZYv9VYPY28AGMYenoQ7Ip-pYlh3A6OAFE,1088
|
|
122
|
+
corekit/integrations/normalizer.py,sha256=9ySWEFom_qdAj3XoCscP7hZr8bYp19q2bOuInitsjbk,670
|
|
123
|
+
corekit/integrations/pipeline.py,sha256=ygyq1l9jsw2SaNfdDYVIg3SKH4UJFz81VjRM8oR2rmM,4644
|
|
124
|
+
corekit/integrations/schemas.py,sha256=m-8dO-Auc4198KrAjl5mynlkMwef_yN7AhbkUDAhmcg,1005
|
|
119
125
|
corekit/jobs/__init__.py,sha256=vsv8o23ApT6MlOwOq-AFYeQW2NT1kjyvu89A5CL2PsY,880
|
|
120
126
|
corekit/jobs/registry.py,sha256=cco6884eVuIYHEtSv6P62MICFjX-QhuSZIFkEJcQi_M,2557
|
|
121
127
|
corekit/jobs/runner.py,sha256=VyiTrQJT-E8-XuNBh8s1QgLJkKhkpBq_398Fs_03Kow,3321
|
|
@@ -185,8 +191,8 @@ corekit/utils/text.py,sha256=0322DdZGuDGEagTyejuda6QT1jbbxNIO6loC-_AGml0,2311
|
|
|
185
191
|
corekit/utils/time.py,sha256=-iPaETXcgDu7J39Kwe938JyZKhPVjl6a572BBqsqcDQ,2215
|
|
186
192
|
corekit/utils/validators.py,sha256=_HVKo3y264KGNaZ-1RyM1x46VPvu7XKcQkrHWDC9uxs,340
|
|
187
193
|
corekit/utils/void.py,sha256=eXDb2QynuguiSgHGJLdgE08MuRnxBdYebRdMD7s1m9o,178
|
|
188
|
-
python_corekit-0.6.
|
|
189
|
-
python_corekit-0.6.
|
|
190
|
-
python_corekit-0.6.
|
|
191
|
-
python_corekit-0.6.
|
|
192
|
-
python_corekit-0.6.
|
|
194
|
+
python_corekit-0.6.3.dist-info/licenses/LICENSE,sha256=357LYxbxAQZ95q5cV-8JK6j-KNgiE6AteRodjEsm-kE,1072
|
|
195
|
+
python_corekit-0.6.3.dist-info/METADATA,sha256=VPiZlFV9pLiK6v23fnYozj6ZbXTgCK4ZmDn40Hi2EN8,15304
|
|
196
|
+
python_corekit-0.6.3.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
197
|
+
python_corekit-0.6.3.dist-info/top_level.txt,sha256=SDK4o8BoaI47E9tPbiNCbbWoc2y6N_wCjwnIEfr-GWI,8
|
|
198
|
+
python_corekit-0.6.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|