terp-cap-sync 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terp/capabilities/sync/__init__.py +109 -0
- terp/capabilities/sync/jobs.py +70 -0
- terp/capabilities/sync/migrations/versions/cb413566b235_create_sync_tables.py +130 -0
- terp/capabilities/sync/models.py +155 -0
- terp/capabilities/sync/py.typed +0 -0
- terp/capabilities/sync/remote.py +127 -0
- terp/capabilities/sync/router.py +95 -0
- terp/capabilities/sync/schedule.py +45 -0
- terp/capabilities/sync/schemas.py +132 -0
- terp/capabilities/sync/service.py +415 -0
- terp/capabilities/sync/store.py +62 -0
- terp_cap_sync-0.1.0.dist-info/METADATA +7 -0
- terp_cap_sync-0.1.0.dist-info/RECORD +15 -0
- terp_cap_sync-0.1.0.dist-info/WHEEL +4 -0
- terp_cap_sync-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""terp.capabilities.sync — reconcile a local entity against an external system.
|
|
2
|
+
|
|
3
|
+
The headline *consumer* capability of the async design (§14): a maintained, secure-by-default
|
|
4
|
+
sync built **only** on the shipped ports — the jobs seam (:func:`terp.core.enqueue` + a typed
|
|
5
|
+
:class:`~terp.core.JobDefinition`), the durable outbox (retry / dead-letter), and the scheduler
|
|
6
|
+
seam (:class:`~terp.core.ScheduleDefinition`). It adds no engine and changes no ``terp.core``.
|
|
7
|
+
|
|
8
|
+
* An app implements one :class:`SyncSource` per entity type (``pull`` reads System B; ``apply``
|
|
9
|
+
upserts the local row through an audited ``BaseService``) and registers it with
|
|
10
|
+
:func:`register_sync_source` at composition time.
|
|
11
|
+
* It mounts the explicit :data:`module` (a *library* cap — no auto-discovery, since a sync does
|
|
12
|
+
nothing without a source) and declares a schedule via :func:`sync_pull_schedule`.
|
|
13
|
+
* On each tick :data:`SYNC_PULL` runs in a worker: :class:`SyncService` opens a
|
|
14
|
+
:class:`SyncRun`, reconciles each remote record against the :class:`SyncMapping` ledger
|
|
15
|
+
(create / update / unchanged — **at-least-once + idempotent**), appends an immutable
|
|
16
|
+
:class:`SyncRecordLog` line per record, and closes the run with its counts + cursor. The
|
|
17
|
+
admin-only router exposes runs, logs, and mappings read-only.
|
|
18
|
+
|
|
19
|
+
It depends only on ``terp-core`` — never a sibling capability or a broker engine; the app
|
|
20
|
+
composes the durable ``OutboxJobQueue`` (and any broker/scheduler adapter) at ``create_app``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from terp.capabilities.sync.jobs import SYNC_PULL, SYNC_PUSH, SyncJobPayload
|
|
26
|
+
from terp.capabilities.sync.models import (
|
|
27
|
+
ACTION_CREATED,
|
|
28
|
+
ACTION_FAILED,
|
|
29
|
+
ACTION_UNCHANGED,
|
|
30
|
+
ACTION_UPDATED,
|
|
31
|
+
STATUS_FAILED,
|
|
32
|
+
STATUS_RUNNING,
|
|
33
|
+
STATUS_SUCCEEDED,
|
|
34
|
+
STATUS_SYNCED,
|
|
35
|
+
SyncMapping,
|
|
36
|
+
SyncRecordLog,
|
|
37
|
+
SyncRun,
|
|
38
|
+
)
|
|
39
|
+
from terp.capabilities.sync.remote import (
|
|
40
|
+
RemotePage,
|
|
41
|
+
RemoteRecord,
|
|
42
|
+
SyncError,
|
|
43
|
+
SyncSource,
|
|
44
|
+
register_sync_source,
|
|
45
|
+
registered_sync_sources,
|
|
46
|
+
reset_sync_sources,
|
|
47
|
+
resolve_sync_source,
|
|
48
|
+
)
|
|
49
|
+
from terp.capabilities.sync.router import module, router
|
|
50
|
+
from terp.capabilities.sync.schedule import sync_pull_schedule, sync_push_schedule
|
|
51
|
+
from terp.capabilities.sync.schemas import (
|
|
52
|
+
SyncMappingDraft,
|
|
53
|
+
SyncMappingRead,
|
|
54
|
+
SyncMappingUpdate,
|
|
55
|
+
SyncRecordLogRead,
|
|
56
|
+
SyncRunDraft,
|
|
57
|
+
SyncRunRead,
|
|
58
|
+
SyncRunUpdate,
|
|
59
|
+
)
|
|
60
|
+
from terp.capabilities.sync.service import (
|
|
61
|
+
SyncService,
|
|
62
|
+
get_run,
|
|
63
|
+
list_mappings,
|
|
64
|
+
list_record_logs,
|
|
65
|
+
list_runs,
|
|
66
|
+
)
|
|
67
|
+
from terp.capabilities.sync.store import record_sync_log
|
|
68
|
+
|
|
69
|
+
__all__ = [
|
|
70
|
+
"ACTION_CREATED",
|
|
71
|
+
"ACTION_FAILED",
|
|
72
|
+
"ACTION_UNCHANGED",
|
|
73
|
+
"ACTION_UPDATED",
|
|
74
|
+
"STATUS_FAILED",
|
|
75
|
+
"STATUS_RUNNING",
|
|
76
|
+
"STATUS_SUCCEEDED",
|
|
77
|
+
"STATUS_SYNCED",
|
|
78
|
+
"SYNC_PULL",
|
|
79
|
+
"SYNC_PUSH",
|
|
80
|
+
"RemotePage",
|
|
81
|
+
"RemoteRecord",
|
|
82
|
+
"SyncError",
|
|
83
|
+
"SyncJobPayload",
|
|
84
|
+
"SyncMapping",
|
|
85
|
+
"SyncMappingDraft",
|
|
86
|
+
"SyncMappingRead",
|
|
87
|
+
"SyncMappingUpdate",
|
|
88
|
+
"SyncRecordLog",
|
|
89
|
+
"SyncRecordLogRead",
|
|
90
|
+
"SyncRun",
|
|
91
|
+
"SyncRunDraft",
|
|
92
|
+
"SyncRunRead",
|
|
93
|
+
"SyncRunUpdate",
|
|
94
|
+
"SyncService",
|
|
95
|
+
"SyncSource",
|
|
96
|
+
"get_run",
|
|
97
|
+
"list_mappings",
|
|
98
|
+
"list_record_logs",
|
|
99
|
+
"list_runs",
|
|
100
|
+
"module",
|
|
101
|
+
"record_sync_log",
|
|
102
|
+
"register_sync_source",
|
|
103
|
+
"registered_sync_sources",
|
|
104
|
+
"reset_sync_sources",
|
|
105
|
+
"resolve_sync_source",
|
|
106
|
+
"router",
|
|
107
|
+
"sync_pull_schedule",
|
|
108
|
+
"sync_push_schedule",
|
|
109
|
+
]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The sync jobs: ``SYNC_PULL`` (reconcile in ← System B) and ``SYNC_PUSH`` (push out → System B).
|
|
2
|
+
|
|
3
|
+
Both are typed :class:`~terp.core.JobDefinition` catalog constants the ``sync`` module declares
|
|
4
|
+
(``ModuleSpec.jobs``), so mounting the module registers them — an app then triggers them on the
|
|
5
|
+
scheduler seam or by hand (``terp jobs run sync.pull ...``). The handler runs in a **worker,
|
|
6
|
+
post-commit**: it resolves the registered :class:`~terp.capabilities.sync.remote.SyncSource` by
|
|
7
|
+
``entity_type`` (a name crosses the wire, never a closure) and drives the audited reconcile. The
|
|
8
|
+
external System-B read lives here in the handler — never in an ``_after_write`` hook (the
|
|
9
|
+
dual-write hazard the design forbids).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import uuid
|
|
15
|
+
|
|
16
|
+
from sqlmodel import Field
|
|
17
|
+
|
|
18
|
+
from terp.core import BaseSchema, JobContext, JobDefinition
|
|
19
|
+
|
|
20
|
+
from terp.capabilities.sync.remote import resolve_sync_source
|
|
21
|
+
from terp.capabilities.sync.service import SyncService
|
|
22
|
+
|
|
23
|
+
_TYPE_MAX = 128
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SyncJobPayload(BaseSchema):
|
|
27
|
+
"""Which entity type to reconcile — the registered source is resolved by this name."""
|
|
28
|
+
|
|
29
|
+
entity_type: str = Field(max_length=_TYPE_MAX)
|
|
30
|
+
tenant_id: uuid.UUID | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _tenant_for_job(ctx: JobContext, payload: SyncJobPayload) -> uuid.UUID | None:
|
|
34
|
+
"""Tenant metadata carried by the payload, falling back to the job envelope context."""
|
|
35
|
+
context_tenant = getattr(ctx, "tenant_id", None)
|
|
36
|
+
if context_tenant is not None:
|
|
37
|
+
return context_tenant
|
|
38
|
+
tenant = payload.model_dump().get("tenant_id")
|
|
39
|
+
if tenant is None:
|
|
40
|
+
return None
|
|
41
|
+
return tenant if isinstance(tenant, uuid.UUID) else uuid.UUID(str(tenant))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _run_pull(ctx: JobContext, payload: SyncJobPayload) -> None:
|
|
45
|
+
"""Reconcile System B → local for the payload's entity type (the ``SYNC_PULL`` handler)."""
|
|
46
|
+
SyncService().pull(
|
|
47
|
+
ctx.session,
|
|
48
|
+
resolve_sync_source(payload.entity_type),
|
|
49
|
+
tenant_id=_tenant_for_job(ctx, payload),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _run_push(ctx: JobContext, payload: SyncJobPayload) -> None:
|
|
54
|
+
"""Push local changes → System B for the payload's entity type (the ``SYNC_PUSH`` handler)."""
|
|
55
|
+
SyncService().push(
|
|
56
|
+
ctx.session,
|
|
57
|
+
resolve_sync_source(payload.entity_type),
|
|
58
|
+
tenant_id=_tenant_for_job(ctx, payload),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
SYNC_PULL = JobDefinition(
|
|
63
|
+
name="sync.pull", payload_schema=SyncJobPayload, handler=_run_pull
|
|
64
|
+
)
|
|
65
|
+
SYNC_PUSH = JobDefinition(
|
|
66
|
+
name="sync.push", payload_schema=SyncJobPayload, handler=_run_push
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
__all__ = ["SYNC_PULL", "SYNC_PUSH", "SyncJobPayload"]
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""create sync tables
|
|
2
|
+
|
|
3
|
+
Revision ID: cb413566b235
|
|
4
|
+
Revises:
|
|
5
|
+
Create Date: 2026-07-01 22:55:48.247941
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Sequence
|
|
11
|
+
|
|
12
|
+
from alembic import op
|
|
13
|
+
import sqlalchemy as sa
|
|
14
|
+
import sqlmodel
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# revision identifiers, used by Alembic.
|
|
18
|
+
revision: str = 'cb413566b235'
|
|
19
|
+
down_revision: str | None = None
|
|
20
|
+
branch_labels: str | Sequence[str] | None = None
|
|
21
|
+
depends_on: str | Sequence[str] | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def upgrade() -> None:
|
|
25
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
26
|
+
op.create_table('sync_mapping',
|
|
27
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
28
|
+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
|
29
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
30
|
+
sa.Column('version', sa.Integer(), nullable=False),
|
|
31
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
32
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
33
|
+
sa.Column('entity_type', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
34
|
+
sa.Column('local_id', sa.Uuid(), nullable=False),
|
|
35
|
+
sa.Column('remote_id', sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
|
|
36
|
+
sa.Column('remote_checksum', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
37
|
+
sa.Column('status', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
38
|
+
sa.Column('last_synced_at', sa.DateTime(timezone=True), nullable=False),
|
|
39
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_mapping')),
|
|
40
|
+
sa.UniqueConstraint('tenant_scope', 'entity_type', 'local_id', name='uq_sync_mapping_local'),
|
|
41
|
+
sa.UniqueConstraint('tenant_scope', 'entity_type', 'remote_id', name='uq_sync_mapping_remote')
|
|
42
|
+
)
|
|
43
|
+
with op.batch_alter_table('sync_mapping', schema=None) as batch_op:
|
|
44
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_entity_type'), ['entity_type'], unique=False)
|
|
45
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_local_id'), ['local_id'], unique=False)
|
|
46
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_remote_id'), ['remote_id'], unique=False)
|
|
47
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_status'), ['status'], unique=False)
|
|
48
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_tenant_id'), ['tenant_id'], unique=False)
|
|
49
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_tenant_scope'), ['tenant_scope'], unique=False)
|
|
50
|
+
|
|
51
|
+
op.create_table('sync_record_log',
|
|
52
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
53
|
+
sa.Column('run_id', sa.Uuid(), nullable=False),
|
|
54
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
55
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
56
|
+
sa.Column('entity_type', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
57
|
+
sa.Column('remote_id', sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
|
|
58
|
+
sa.Column('action', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
59
|
+
sa.Column('message', sqlmodel.sql.sqltypes.AutoString(length=2000), nullable=True),
|
|
60
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
61
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_record_log'))
|
|
62
|
+
)
|
|
63
|
+
with op.batch_alter_table('sync_record_log', schema=None) as batch_op:
|
|
64
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_action'), ['action'], unique=False)
|
|
65
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_created_at'), ['created_at'], unique=False)
|
|
66
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_entity_type'), ['entity_type'], unique=False)
|
|
67
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_remote_id'), ['remote_id'], unique=False)
|
|
68
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_run_id'), ['run_id'], unique=False)
|
|
69
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_tenant_id'), ['tenant_id'], unique=False)
|
|
70
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_tenant_scope'), ['tenant_scope'], unique=False)
|
|
71
|
+
|
|
72
|
+
op.create_table('sync_run',
|
|
73
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
74
|
+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
|
75
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
76
|
+
sa.Column('version', sa.Integer(), nullable=False),
|
|
77
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
78
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
79
|
+
sa.Column('source', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
80
|
+
sa.Column('status', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
81
|
+
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
|
82
|
+
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
|
83
|
+
sa.Column('processed_count', sa.Integer(), nullable=False),
|
|
84
|
+
sa.Column('created_count', sa.Integer(), nullable=False),
|
|
85
|
+
sa.Column('updated_count', sa.Integer(), nullable=False),
|
|
86
|
+
sa.Column('failed_count', sa.Integer(), nullable=False),
|
|
87
|
+
sa.Column('cursor', sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True),
|
|
88
|
+
sa.Column('error', sqlmodel.sql.sqltypes.AutoString(length=2000), nullable=True),
|
|
89
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_run'))
|
|
90
|
+
)
|
|
91
|
+
with op.batch_alter_table('sync_run', schema=None) as batch_op:
|
|
92
|
+
batch_op.create_index(batch_op.f('ix_sync_run_source'), ['source'], unique=False)
|
|
93
|
+
batch_op.create_index(batch_op.f('ix_sync_run_started_at'), ['started_at'], unique=False)
|
|
94
|
+
batch_op.create_index(batch_op.f('ix_sync_run_status'), ['status'], unique=False)
|
|
95
|
+
batch_op.create_index(batch_op.f('ix_sync_run_tenant_id'), ['tenant_id'], unique=False)
|
|
96
|
+
batch_op.create_index(batch_op.f('ix_sync_run_tenant_scope'), ['tenant_scope'], unique=False)
|
|
97
|
+
|
|
98
|
+
# ### end Alembic commands ###
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def downgrade() -> None:
|
|
102
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
103
|
+
with op.batch_alter_table('sync_run', schema=None) as batch_op:
|
|
104
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_tenant_scope'))
|
|
105
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_tenant_id'))
|
|
106
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_status'))
|
|
107
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_started_at'))
|
|
108
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_source'))
|
|
109
|
+
|
|
110
|
+
op.drop_table('sync_run')
|
|
111
|
+
with op.batch_alter_table('sync_record_log', schema=None) as batch_op:
|
|
112
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_tenant_scope'))
|
|
113
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_tenant_id'))
|
|
114
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_run_id'))
|
|
115
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_remote_id'))
|
|
116
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_entity_type'))
|
|
117
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_created_at'))
|
|
118
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_action'))
|
|
119
|
+
|
|
120
|
+
op.drop_table('sync_record_log')
|
|
121
|
+
with op.batch_alter_table('sync_mapping', schema=None) as batch_op:
|
|
122
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_tenant_scope'))
|
|
123
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_tenant_id'))
|
|
124
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_status'))
|
|
125
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_remote_id'))
|
|
126
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_local_id'))
|
|
127
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_entity_type'))
|
|
128
|
+
|
|
129
|
+
op.drop_table('sync_mapping')
|
|
130
|
+
# ### end Alembic commands ###
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Sync tables: the identity ledger, per-run aggregates, and an append-only record log.
|
|
2
|
+
|
|
3
|
+
A sync reconciles a local entity type against an external system. Three tables back it:
|
|
4
|
+
|
|
5
|
+
* :class:`SyncMapping` (``BaseTable``) — the identity ledger tying a local row to its remote
|
|
6
|
+
counterpart. Unique on (``tenant_scope``, ``entity_type``, ``local_id``) **and**
|
|
7
|
+
(``tenant_scope``, ``entity_type``, ``remote_id``), so an upsert is idempotent from either
|
|
8
|
+
side without colliding across tenants — the natural at-least-once dedupe key the design (§6
|
|
9
|
+
rule 3) leans on.
|
|
10
|
+
* :class:`SyncRun` (``BaseTable``) — one reconcile attempt's aggregates (counts + the
|
|
11
|
+
high-watermark ``cursor``), stored so a stats view never pays a per-row ``COUNT(*)``
|
|
12
|
+
(review M5).
|
|
13
|
+
* :class:`SyncRecordLog` — one append-only line per record processed, immutable exactly like
|
|
14
|
+
:class:`~terp.capabilities.audit.AuditEvent` (``UUIDPrimaryKeyMixin``, no OCC ``version`` /
|
|
15
|
+
``updated_at``); high-volume, so plan retention.
|
|
16
|
+
|
|
17
|
+
Every caller-influenceable ``str`` column caps its length.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import uuid
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
from typing import Final
|
|
25
|
+
|
|
26
|
+
from sqlalchemy import DateTime, UniqueConstraint
|
|
27
|
+
from sqlmodel import Field, SQLModel
|
|
28
|
+
|
|
29
|
+
from terp.core import BaseTable, UUIDPrimaryKeyMixin
|
|
30
|
+
|
|
31
|
+
# Mapping/run status + record-log actions (plain str columns, dependency-light leaves like
|
|
32
|
+
# ``AuditEvent.action`` — never a higher-layer enum on the table).
|
|
33
|
+
STATUS_SYNCED: Final[str] = "synced"
|
|
34
|
+
STATUS_RUNNING: Final[str] = "running"
|
|
35
|
+
STATUS_SUCCEEDED: Final[str] = "succeeded"
|
|
36
|
+
STATUS_FAILED: Final[str] = "failed"
|
|
37
|
+
|
|
38
|
+
ACTION_CREATED: Final[str] = "created"
|
|
39
|
+
ACTION_UPDATED: Final[str] = "updated"
|
|
40
|
+
ACTION_UNCHANGED: Final[str] = "unchanged"
|
|
41
|
+
ACTION_FAILED: Final[str] = "failed"
|
|
42
|
+
|
|
43
|
+
# Hard caps so a hostile / oversized value can never break the INSERT or the ledger.
|
|
44
|
+
_TYPE_MAX: Final[int] = 128
|
|
45
|
+
_REMOTE_ID_MAX: Final[int] = 200
|
|
46
|
+
_CHECKSUM_MAX: Final[int] = 128
|
|
47
|
+
_STATUS_MAX: Final[int] = 16
|
|
48
|
+
_ACTION_MAX: Final[int] = 16
|
|
49
|
+
_CURSOR_MAX: Final[int] = 512
|
|
50
|
+
_MESSAGE_MAX: Final[int] = 2000
|
|
51
|
+
_GLOBAL_TENANT_SCOPE: Final[str] = "global"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _utc_now() -> datetime:
|
|
55
|
+
"""UTC ``now`` provider for the timestamp columns."""
|
|
56
|
+
return datetime.now(UTC)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class SyncMapping(BaseTable, table=True):
|
|
60
|
+
"""The identity ledger: one local row ↔ one remote row for an ``entity_type``.
|
|
61
|
+
|
|
62
|
+
``id`` / ``created_at`` / ``updated_at`` / ``version`` are inherited from ``BaseTable``.
|
|
63
|
+
The two unique constraints make the mapping the idempotent upsert key from either side, so
|
|
64
|
+
an at-least-once redelivery of the same remote record never double-creates a local row.
|
|
65
|
+
``remote_checksum`` lets the reconcile detect a change without deep-diffing the payload.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
__tablename__ = "sync_mapping"
|
|
69
|
+
__table_args__ = (
|
|
70
|
+
UniqueConstraint(
|
|
71
|
+
"tenant_scope", "entity_type", "local_id", name="uq_sync_mapping_local"
|
|
72
|
+
),
|
|
73
|
+
UniqueConstraint(
|
|
74
|
+
"tenant_scope", "entity_type", "remote_id", name="uq_sync_mapping_remote"
|
|
75
|
+
),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
79
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
80
|
+
entity_type: str = Field(max_length=_TYPE_MAX, index=True)
|
|
81
|
+
local_id: uuid.UUID = Field(index=True)
|
|
82
|
+
remote_id: str = Field(max_length=_REMOTE_ID_MAX, index=True)
|
|
83
|
+
remote_checksum: str = Field(max_length=_CHECKSUM_MAX)
|
|
84
|
+
status: str = Field(default=STATUS_SYNCED, max_length=_STATUS_MAX, index=True)
|
|
85
|
+
last_synced_at: datetime = Field(
|
|
86
|
+
default_factory=_utc_now,
|
|
87
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
88
|
+
nullable=False,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class SyncRun(BaseTable, table=True):
|
|
93
|
+
"""One reconcile attempt for a ``source`` (entity type): status, counts, and cursor.
|
|
94
|
+
|
|
95
|
+
Aggregates are stored here (``processed_count`` / ``created_count`` / ``updated_count`` /
|
|
96
|
+
``failed_count``) so a stats view reads a single row instead of counting the log; the
|
|
97
|
+
high-watermark ``cursor`` is where the next run resumes.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
__tablename__ = "sync_run"
|
|
101
|
+
|
|
102
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
103
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
104
|
+
source: str = Field(max_length=_TYPE_MAX, index=True)
|
|
105
|
+
status: str = Field(default=STATUS_RUNNING, max_length=_STATUS_MAX, index=True)
|
|
106
|
+
started_at: datetime = Field(
|
|
107
|
+
default_factory=_utc_now,
|
|
108
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
109
|
+
nullable=False,
|
|
110
|
+
index=True,
|
|
111
|
+
)
|
|
112
|
+
finished_at: datetime | None = Field(
|
|
113
|
+
default=None,
|
|
114
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
115
|
+
nullable=True,
|
|
116
|
+
)
|
|
117
|
+
processed_count: int = Field(default=0)
|
|
118
|
+
created_count: int = Field(default=0)
|
|
119
|
+
updated_count: int = Field(default=0)
|
|
120
|
+
failed_count: int = Field(default=0)
|
|
121
|
+
cursor: str | None = Field(default=None, max_length=_CURSOR_MAX)
|
|
122
|
+
error: str | None = Field(default=None, max_length=_MESSAGE_MAX)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class SyncRecordLog(UUIDPrimaryKeyMixin, SQLModel, table=True): # arch-allow-table-models-use-base-table: append-only per-record log (like AuditEvent) — immutable, no version/updated_at by design (see module docstring)
|
|
126
|
+
__tablename__ = "sync_record_log"
|
|
127
|
+
|
|
128
|
+
run_id: uuid.UUID = Field(index=True)
|
|
129
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
130
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
131
|
+
entity_type: str = Field(max_length=_TYPE_MAX, index=True)
|
|
132
|
+
remote_id: str = Field(max_length=_REMOTE_ID_MAX, index=True)
|
|
133
|
+
action: str = Field(max_length=_ACTION_MAX, index=True)
|
|
134
|
+
message: str | None = Field(default=None, max_length=_MESSAGE_MAX)
|
|
135
|
+
created_at: datetime = Field(
|
|
136
|
+
default_factory=_utc_now,
|
|
137
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
138
|
+
nullable=False,
|
|
139
|
+
index=True,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
__all__ = [
|
|
144
|
+
"ACTION_CREATED",
|
|
145
|
+
"ACTION_FAILED",
|
|
146
|
+
"ACTION_UNCHANGED",
|
|
147
|
+
"ACTION_UPDATED",
|
|
148
|
+
"STATUS_FAILED",
|
|
149
|
+
"STATUS_RUNNING",
|
|
150
|
+
"STATUS_SUCCEEDED",
|
|
151
|
+
"STATUS_SYNCED",
|
|
152
|
+
"SyncMapping",
|
|
153
|
+
"SyncRecordLog",
|
|
154
|
+
"SyncRun",
|
|
155
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""The ``SyncSource`` seam: how a sync reads System B and applies records to the local target.
|
|
2
|
+
|
|
3
|
+
An app implements one :class:`SyncSource` per entity type and registers it; the reconcile
|
|
4
|
+
engine (:class:`~terp.capabilities.sync.SyncService`) owns everything else — the run
|
|
5
|
+
bookkeeping, the mapping ledger, the record log. This is the seam that keeps the two
|
|
6
|
+
design invariants (§14):
|
|
7
|
+
|
|
8
|
+
* the **external read** of System B happens in :meth:`SyncSource.pull`, which the reconcile
|
|
9
|
+
calls **inside the job handler** (a worker, post-commit) — never in an ``_after_write`` hook
|
|
10
|
+
(the dual-write hazard the ADR-0040 review flagged); and
|
|
11
|
+
* the **local write** happens in :meth:`SyncSource.apply`, which MUST go through an audited
|
|
12
|
+
``BaseService`` so a synced row is actor / owner stamped and audited like any other write.
|
|
13
|
+
|
|
14
|
+
The source is resolved by ``entity_type`` from a small registry (the job carries the entity
|
|
15
|
+
type, not a closure — a remote worker resolves the source by name, like a job handler).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import uuid
|
|
21
|
+
from abc import ABC, abstractmethod
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from sqlmodel import Session
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SyncError(RuntimeError):
|
|
30
|
+
"""Raised when a sync source is unregistered, or a direction it does not support is run."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class RemoteRecord:
|
|
35
|
+
"""One record from System B: its remote id, a change-detecting checksum, and the payload.
|
|
36
|
+
|
|
37
|
+
``payload`` is plain JSON scalars (ids, not entities) — it crosses the job boundary and is
|
|
38
|
+
handed to :meth:`SyncSource.apply` to upsert the local row. ``checksum`` lets the reconcile
|
|
39
|
+
skip an unchanged record without deep-diffing.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
remote_id: str
|
|
43
|
+
checksum: str
|
|
44
|
+
payload: Mapping[str, Any] = field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class RemotePage:
|
|
49
|
+
"""One page of remote records plus the cursor to resume from (the high-watermark)."""
|
|
50
|
+
|
|
51
|
+
records: tuple[RemoteRecord, ...] = ()
|
|
52
|
+
next_cursor: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class SyncSource(ABC):
|
|
56
|
+
"""A registered source/target pair for one ``entity_type`` — the app implements it.
|
|
57
|
+
|
|
58
|
+
Set ``entity_type`` on the subclass; implement :meth:`pull` (read System B) and
|
|
59
|
+
:meth:`apply` (upsert the local row via an audited ``BaseService``). Override :meth:`push`
|
|
60
|
+
only for a source that also pushes local changes outward (``SYNC_PUSH``); the default
|
|
61
|
+
fails closed.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
entity_type: str
|
|
65
|
+
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def pull(self, cursor: str | None) -> RemotePage:
|
|
68
|
+
"""Read one page of remote records from *cursor* onward (the external System-B call)."""
|
|
69
|
+
|
|
70
|
+
@abstractmethod
|
|
71
|
+
def apply(
|
|
72
|
+
self, session: Session, record: RemoteRecord, local_id: uuid.UUID | None
|
|
73
|
+
) -> uuid.UUID:
|
|
74
|
+
"""Create (``local_id`` is ``None``) or update the local entity; return its id.
|
|
75
|
+
|
|
76
|
+
MUST persist through an audited ``BaseService`` so the synced row is stamped + audited.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
def push(self, session: Session) -> int: # noqa: ARG002 - default seam; overridden by push-capable sources
|
|
80
|
+
"""Push local changes to System B (``SYNC_PUSH``); default: unsupported. Returns count."""
|
|
81
|
+
raise SyncError(f"sync source {self.entity_type!r} does not implement push")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_sources: dict[str, SyncSource] = {}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def register_sync_source(source: SyncSource) -> None:
|
|
88
|
+
"""Register *source* for its ``entity_type`` (replacing any prior registration).
|
|
89
|
+
|
|
90
|
+
A capability registration (like a scope predicate): it persists across composed apps and is
|
|
91
|
+
cleared only by :func:`reset_sync_sources`. An app calls this at composition time for each
|
|
92
|
+
entity type it syncs.
|
|
93
|
+
"""
|
|
94
|
+
_sources[source.entity_type] = source
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def resolve_sync_source(entity_type: str) -> SyncSource:
|
|
98
|
+
"""Return the source registered for *entity_type*, or fail closed with :class:`SyncError`."""
|
|
99
|
+
try:
|
|
100
|
+
return _sources[entity_type]
|
|
101
|
+
except KeyError:
|
|
102
|
+
raise SyncError(
|
|
103
|
+
f"no sync source registered for entity type {entity_type!r}; "
|
|
104
|
+
"call register_sync_source(...) at composition time"
|
|
105
|
+
) from None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def registered_sync_sources() -> tuple[str, ...]:
|
|
109
|
+
"""The entity types with a registered source, in registration order."""
|
|
110
|
+
return tuple(_sources)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def reset_sync_sources() -> None:
|
|
114
|
+
"""Clear the sync-source registry (a test seam; apps re-register at composition)."""
|
|
115
|
+
_sources.clear()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
__all__ = [
|
|
119
|
+
"RemotePage",
|
|
120
|
+
"RemoteRecord",
|
|
121
|
+
"SyncError",
|
|
122
|
+
"SyncSource",
|
|
123
|
+
"register_sync_source",
|
|
124
|
+
"registered_sync_sources",
|
|
125
|
+
"reset_sync_sources",
|
|
126
|
+
"resolve_sync_source",
|
|
127
|
+
]
|