terp-cap-sync 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terp_cap_sync-0.1.0/.gitignore +47 -0
- terp_cap_sync-0.1.0/PKG-INFO +7 -0
- terp_cap_sync-0.1.0/escape-hatch-budget.json +5 -0
- terp_cap_sync-0.1.0/pyproject.toml +31 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/__init__.py +109 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/jobs.py +70 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/migrations/versions/cb413566b235_create_sync_tables.py +130 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/models.py +155 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/py.typed +0 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/remote.py +127 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/router.py +95 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/schedule.py +45 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/schemas.py +132 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/service.py +415 -0
- terp_cap_sync-0.1.0/src/terp/capabilities/sync/store.py +62 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
.venv-*/
|
|
10
|
+
venv/
|
|
11
|
+
.pytest_cache/
|
|
12
|
+
.mypy_cache/
|
|
13
|
+
.ruff_cache/
|
|
14
|
+
.coverage
|
|
15
|
+
htmlcov/
|
|
16
|
+
|
|
17
|
+
# uv
|
|
18
|
+
uv.lock
|
|
19
|
+
|
|
20
|
+
# Node
|
|
21
|
+
node_modules/
|
|
22
|
+
.pnpm-store/
|
|
23
|
+
*.tsbuildinfo
|
|
24
|
+
|
|
25
|
+
# Playwright (conformance e2e) artifacts
|
|
26
|
+
test-results/
|
|
27
|
+
playwright-report/
|
|
28
|
+
blob-report/
|
|
29
|
+
playwright/.cache/
|
|
30
|
+
.last-run.json
|
|
31
|
+
|
|
32
|
+
# Local frontend template render checks
|
|
33
|
+
apps/example/_frontend_tpl_check/
|
|
34
|
+
|
|
35
|
+
# Editor / OS
|
|
36
|
+
.DS_Store
|
|
37
|
+
.idea/
|
|
38
|
+
*.local
|
|
39
|
+
|
|
40
|
+
# Local environment overrides — never commit (a real .env may hold SECRET_KEY).
|
|
41
|
+
# The tracked template is `.env.example`.
|
|
42
|
+
.env
|
|
43
|
+
.env.*
|
|
44
|
+
!.env.example
|
|
45
|
+
!.env.example.jinja
|
|
46
|
+
# Rendered app-declared variables (environment.schema.json) — may hold secrets.
|
|
47
|
+
.app.env
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "terp-cap-sync"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Terp sync capability — reconcile a local entity against an external system on the jobs/scheduler seam."
|
|
9
|
+
requires-python = ">=3.13"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"terp-core==0.1.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
# A LIBRARY capability (like terp-cap-identity): it owns tables + a mountable read router +
|
|
16
|
+
# the reconcile engine, but declares NO `terp.capabilities` auto-discovery entry point. A
|
|
17
|
+
# sync does nothing until an app registers a concrete SyncSource, so the app opts in
|
|
18
|
+
# explicitly — it includes the exported `module` in its create_app(specs=[...]) and calls
|
|
19
|
+
# register_sync_source(...). Auto-mounting it in an app with no SyncSource would only expose
|
|
20
|
+
# hollow endpoints, so discovery is deliberately explicit.
|
|
21
|
+
|
|
22
|
+
# Owns the `sync_mapping` / `sync_run` / append-only `sync_record_log` tables, so it ships an
|
|
23
|
+
# independent, linear Alembic history (its own `alembic_version_sync` table). `terp migrate`
|
|
24
|
+
# discovers this via the `terp.migrations` group (ADR 0027).
|
|
25
|
+
[project.entry-points."terp.migrations"]
|
|
26
|
+
sync = "terp.capabilities.sync"
|
|
27
|
+
|
|
28
|
+
# PEP 420 namespace package: this distribution owns only `terp.capabilities.sync`.
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
sources = ["src"]
|
|
31
|
+
only-include = ["src/terp/capabilities/sync"]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""terp.capabilities.sync — reconcile a local entity against an external system.
|
|
2
|
+
|
|
3
|
+
The headline *consumer* capability of the async design (§14): a maintained, secure-by-default
|
|
4
|
+
sync built **only** on the shipped ports — the jobs seam (:func:`terp.core.enqueue` + a typed
|
|
5
|
+
:class:`~terp.core.JobDefinition`), the durable outbox (retry / dead-letter), and the scheduler
|
|
6
|
+
seam (:class:`~terp.core.ScheduleDefinition`). It adds no engine and changes no ``terp.core``.
|
|
7
|
+
|
|
8
|
+
* An app implements one :class:`SyncSource` per entity type (``pull`` reads System B; ``apply``
|
|
9
|
+
upserts the local row through an audited ``BaseService``) and registers it with
|
|
10
|
+
:func:`register_sync_source` at composition time.
|
|
11
|
+
* It mounts the explicit :data:`module` (a *library* cap — no auto-discovery, since a sync does
|
|
12
|
+
nothing without a source) and declares a schedule via :func:`sync_pull_schedule`.
|
|
13
|
+
* On each tick :data:`SYNC_PULL` runs in a worker: :class:`SyncService` opens a
|
|
14
|
+
:class:`SyncRun`, reconciles each remote record against the :class:`SyncMapping` ledger
|
|
15
|
+
(create / update / unchanged — **at-least-once + idempotent**), appends an immutable
|
|
16
|
+
:class:`SyncRecordLog` line per record, and closes the run with its counts + cursor. The
|
|
17
|
+
admin-only router exposes runs, logs, and mappings read-only.
|
|
18
|
+
|
|
19
|
+
It depends only on ``terp-core`` — never a sibling capability or a broker engine; the app
|
|
20
|
+
composes the durable ``OutboxJobQueue`` (and any broker/scheduler adapter) at ``create_app``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from terp.capabilities.sync.jobs import SYNC_PULL, SYNC_PUSH, SyncJobPayload
|
|
26
|
+
from terp.capabilities.sync.models import (
|
|
27
|
+
ACTION_CREATED,
|
|
28
|
+
ACTION_FAILED,
|
|
29
|
+
ACTION_UNCHANGED,
|
|
30
|
+
ACTION_UPDATED,
|
|
31
|
+
STATUS_FAILED,
|
|
32
|
+
STATUS_RUNNING,
|
|
33
|
+
STATUS_SUCCEEDED,
|
|
34
|
+
STATUS_SYNCED,
|
|
35
|
+
SyncMapping,
|
|
36
|
+
SyncRecordLog,
|
|
37
|
+
SyncRun,
|
|
38
|
+
)
|
|
39
|
+
from terp.capabilities.sync.remote import (
|
|
40
|
+
RemotePage,
|
|
41
|
+
RemoteRecord,
|
|
42
|
+
SyncError,
|
|
43
|
+
SyncSource,
|
|
44
|
+
register_sync_source,
|
|
45
|
+
registered_sync_sources,
|
|
46
|
+
reset_sync_sources,
|
|
47
|
+
resolve_sync_source,
|
|
48
|
+
)
|
|
49
|
+
from terp.capabilities.sync.router import module, router
|
|
50
|
+
from terp.capabilities.sync.schedule import sync_pull_schedule, sync_push_schedule
|
|
51
|
+
from terp.capabilities.sync.schemas import (
|
|
52
|
+
SyncMappingDraft,
|
|
53
|
+
SyncMappingRead,
|
|
54
|
+
SyncMappingUpdate,
|
|
55
|
+
SyncRecordLogRead,
|
|
56
|
+
SyncRunDraft,
|
|
57
|
+
SyncRunRead,
|
|
58
|
+
SyncRunUpdate,
|
|
59
|
+
)
|
|
60
|
+
from terp.capabilities.sync.service import (
|
|
61
|
+
SyncService,
|
|
62
|
+
get_run,
|
|
63
|
+
list_mappings,
|
|
64
|
+
list_record_logs,
|
|
65
|
+
list_runs,
|
|
66
|
+
)
|
|
67
|
+
from terp.capabilities.sync.store import record_sync_log
|
|
68
|
+
|
|
69
|
+
__all__ = [
|
|
70
|
+
"ACTION_CREATED",
|
|
71
|
+
"ACTION_FAILED",
|
|
72
|
+
"ACTION_UNCHANGED",
|
|
73
|
+
"ACTION_UPDATED",
|
|
74
|
+
"STATUS_FAILED",
|
|
75
|
+
"STATUS_RUNNING",
|
|
76
|
+
"STATUS_SUCCEEDED",
|
|
77
|
+
"STATUS_SYNCED",
|
|
78
|
+
"SYNC_PULL",
|
|
79
|
+
"SYNC_PUSH",
|
|
80
|
+
"RemotePage",
|
|
81
|
+
"RemoteRecord",
|
|
82
|
+
"SyncError",
|
|
83
|
+
"SyncJobPayload",
|
|
84
|
+
"SyncMapping",
|
|
85
|
+
"SyncMappingDraft",
|
|
86
|
+
"SyncMappingRead",
|
|
87
|
+
"SyncMappingUpdate",
|
|
88
|
+
"SyncRecordLog",
|
|
89
|
+
"SyncRecordLogRead",
|
|
90
|
+
"SyncRun",
|
|
91
|
+
"SyncRunDraft",
|
|
92
|
+
"SyncRunRead",
|
|
93
|
+
"SyncRunUpdate",
|
|
94
|
+
"SyncService",
|
|
95
|
+
"SyncSource",
|
|
96
|
+
"get_run",
|
|
97
|
+
"list_mappings",
|
|
98
|
+
"list_record_logs",
|
|
99
|
+
"list_runs",
|
|
100
|
+
"module",
|
|
101
|
+
"record_sync_log",
|
|
102
|
+
"register_sync_source",
|
|
103
|
+
"registered_sync_sources",
|
|
104
|
+
"reset_sync_sources",
|
|
105
|
+
"resolve_sync_source",
|
|
106
|
+
"router",
|
|
107
|
+
"sync_pull_schedule",
|
|
108
|
+
"sync_push_schedule",
|
|
109
|
+
]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The sync jobs: ``SYNC_PULL`` (reconcile in ← System B) and ``SYNC_PUSH`` (push out → System B).
|
|
2
|
+
|
|
3
|
+
Both are typed :class:`~terp.core.JobDefinition` catalog constants the ``sync`` module declares
|
|
4
|
+
(``ModuleSpec.jobs``), so mounting the module registers them — an app then triggers them on the
|
|
5
|
+
scheduler seam or by hand (``terp jobs run sync.pull ...``). The handler runs in a **worker,
|
|
6
|
+
post-commit**: it resolves the registered :class:`~terp.capabilities.sync.remote.SyncSource` by
|
|
7
|
+
``entity_type`` (a name crosses the wire, never a closure) and drives the audited reconcile. The
|
|
8
|
+
external System-B read lives here in the handler — never in an ``_after_write`` hook (the
|
|
9
|
+
dual-write hazard the design forbids).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import uuid
|
|
15
|
+
|
|
16
|
+
from sqlmodel import Field
|
|
17
|
+
|
|
18
|
+
from terp.core import BaseSchema, JobContext, JobDefinition
|
|
19
|
+
|
|
20
|
+
from terp.capabilities.sync.remote import resolve_sync_source
|
|
21
|
+
from terp.capabilities.sync.service import SyncService
|
|
22
|
+
|
|
23
|
+
_TYPE_MAX = 128
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SyncJobPayload(BaseSchema):
|
|
27
|
+
"""Which entity type to reconcile — the registered source is resolved by this name."""
|
|
28
|
+
|
|
29
|
+
entity_type: str = Field(max_length=_TYPE_MAX)
|
|
30
|
+
tenant_id: uuid.UUID | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _tenant_for_job(ctx: JobContext, payload: SyncJobPayload) -> uuid.UUID | None:
|
|
34
|
+
"""Tenant metadata carried by the payload, falling back to the job envelope context."""
|
|
35
|
+
context_tenant = getattr(ctx, "tenant_id", None)
|
|
36
|
+
if context_tenant is not None:
|
|
37
|
+
return context_tenant
|
|
38
|
+
tenant = payload.model_dump().get("tenant_id")
|
|
39
|
+
if tenant is None:
|
|
40
|
+
return None
|
|
41
|
+
return tenant if isinstance(tenant, uuid.UUID) else uuid.UUID(str(tenant))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _run_pull(ctx: JobContext, payload: SyncJobPayload) -> None:
|
|
45
|
+
"""Reconcile System B → local for the payload's entity type (the ``SYNC_PULL`` handler)."""
|
|
46
|
+
SyncService().pull(
|
|
47
|
+
ctx.session,
|
|
48
|
+
resolve_sync_source(payload.entity_type),
|
|
49
|
+
tenant_id=_tenant_for_job(ctx, payload),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _run_push(ctx: JobContext, payload: SyncJobPayload) -> None:
|
|
54
|
+
"""Push local changes → System B for the payload's entity type (the ``SYNC_PUSH`` handler)."""
|
|
55
|
+
SyncService().push(
|
|
56
|
+
ctx.session,
|
|
57
|
+
resolve_sync_source(payload.entity_type),
|
|
58
|
+
tenant_id=_tenant_for_job(ctx, payload),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
SYNC_PULL = JobDefinition(
|
|
63
|
+
name="sync.pull", payload_schema=SyncJobPayload, handler=_run_pull
|
|
64
|
+
)
|
|
65
|
+
SYNC_PUSH = JobDefinition(
|
|
66
|
+
name="sync.push", payload_schema=SyncJobPayload, handler=_run_push
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
__all__ = ["SYNC_PULL", "SYNC_PUSH", "SyncJobPayload"]
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""create sync tables
|
|
2
|
+
|
|
3
|
+
Revision ID: cb413566b235
|
|
4
|
+
Revises:
|
|
5
|
+
Create Date: 2026-07-01 22:55:48.247941
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Sequence
|
|
11
|
+
|
|
12
|
+
from alembic import op
|
|
13
|
+
import sqlalchemy as sa
|
|
14
|
+
import sqlmodel
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# revision identifiers, used by Alembic.
|
|
18
|
+
revision: str = 'cb413566b235'
|
|
19
|
+
down_revision: str | None = None
|
|
20
|
+
branch_labels: str | Sequence[str] | None = None
|
|
21
|
+
depends_on: str | Sequence[str] | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def upgrade() -> None:
|
|
25
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
26
|
+
op.create_table('sync_mapping',
|
|
27
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
28
|
+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
|
29
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
30
|
+
sa.Column('version', sa.Integer(), nullable=False),
|
|
31
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
32
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
33
|
+
sa.Column('entity_type', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
34
|
+
sa.Column('local_id', sa.Uuid(), nullable=False),
|
|
35
|
+
sa.Column('remote_id', sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
|
|
36
|
+
sa.Column('remote_checksum', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
37
|
+
sa.Column('status', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
38
|
+
sa.Column('last_synced_at', sa.DateTime(timezone=True), nullable=False),
|
|
39
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_mapping')),
|
|
40
|
+
sa.UniqueConstraint('tenant_scope', 'entity_type', 'local_id', name='uq_sync_mapping_local'),
|
|
41
|
+
sa.UniqueConstraint('tenant_scope', 'entity_type', 'remote_id', name='uq_sync_mapping_remote')
|
|
42
|
+
)
|
|
43
|
+
with op.batch_alter_table('sync_mapping', schema=None) as batch_op:
|
|
44
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_entity_type'), ['entity_type'], unique=False)
|
|
45
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_local_id'), ['local_id'], unique=False)
|
|
46
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_remote_id'), ['remote_id'], unique=False)
|
|
47
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_status'), ['status'], unique=False)
|
|
48
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_tenant_id'), ['tenant_id'], unique=False)
|
|
49
|
+
batch_op.create_index(batch_op.f('ix_sync_mapping_tenant_scope'), ['tenant_scope'], unique=False)
|
|
50
|
+
|
|
51
|
+
op.create_table('sync_record_log',
|
|
52
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
53
|
+
sa.Column('run_id', sa.Uuid(), nullable=False),
|
|
54
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
55
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
56
|
+
sa.Column('entity_type', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
57
|
+
sa.Column('remote_id', sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
|
|
58
|
+
sa.Column('action', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
59
|
+
sa.Column('message', sqlmodel.sql.sqltypes.AutoString(length=2000), nullable=True),
|
|
60
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
61
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_record_log'))
|
|
62
|
+
)
|
|
63
|
+
with op.batch_alter_table('sync_record_log', schema=None) as batch_op:
|
|
64
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_action'), ['action'], unique=False)
|
|
65
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_created_at'), ['created_at'], unique=False)
|
|
66
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_entity_type'), ['entity_type'], unique=False)
|
|
67
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_remote_id'), ['remote_id'], unique=False)
|
|
68
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_run_id'), ['run_id'], unique=False)
|
|
69
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_tenant_id'), ['tenant_id'], unique=False)
|
|
70
|
+
batch_op.create_index(batch_op.f('ix_sync_record_log_tenant_scope'), ['tenant_scope'], unique=False)
|
|
71
|
+
|
|
72
|
+
op.create_table('sync_run',
|
|
73
|
+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
|
74
|
+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
|
75
|
+
sa.Column('id', sa.Uuid(), nullable=False),
|
|
76
|
+
sa.Column('version', sa.Integer(), nullable=False),
|
|
77
|
+
sa.Column('tenant_scope', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False),
|
|
78
|
+
sa.Column('tenant_id', sa.Uuid(), nullable=True),
|
|
79
|
+
sa.Column('source', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
|
|
80
|
+
sa.Column('status', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False),
|
|
81
|
+
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
|
82
|
+
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
|
83
|
+
sa.Column('processed_count', sa.Integer(), nullable=False),
|
|
84
|
+
sa.Column('created_count', sa.Integer(), nullable=False),
|
|
85
|
+
sa.Column('updated_count', sa.Integer(), nullable=False),
|
|
86
|
+
sa.Column('failed_count', sa.Integer(), nullable=False),
|
|
87
|
+
sa.Column('cursor', sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True),
|
|
88
|
+
sa.Column('error', sqlmodel.sql.sqltypes.AutoString(length=2000), nullable=True),
|
|
89
|
+
sa.PrimaryKeyConstraint('id', name=op.f('pk_sync_run'))
|
|
90
|
+
)
|
|
91
|
+
with op.batch_alter_table('sync_run', schema=None) as batch_op:
|
|
92
|
+
batch_op.create_index(batch_op.f('ix_sync_run_source'), ['source'], unique=False)
|
|
93
|
+
batch_op.create_index(batch_op.f('ix_sync_run_started_at'), ['started_at'], unique=False)
|
|
94
|
+
batch_op.create_index(batch_op.f('ix_sync_run_status'), ['status'], unique=False)
|
|
95
|
+
batch_op.create_index(batch_op.f('ix_sync_run_tenant_id'), ['tenant_id'], unique=False)
|
|
96
|
+
batch_op.create_index(batch_op.f('ix_sync_run_tenant_scope'), ['tenant_scope'], unique=False)
|
|
97
|
+
|
|
98
|
+
# ### end Alembic commands ###
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def downgrade() -> None:
|
|
102
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
|
103
|
+
with op.batch_alter_table('sync_run', schema=None) as batch_op:
|
|
104
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_tenant_scope'))
|
|
105
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_tenant_id'))
|
|
106
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_status'))
|
|
107
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_started_at'))
|
|
108
|
+
batch_op.drop_index(batch_op.f('ix_sync_run_source'))
|
|
109
|
+
|
|
110
|
+
op.drop_table('sync_run')
|
|
111
|
+
with op.batch_alter_table('sync_record_log', schema=None) as batch_op:
|
|
112
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_tenant_scope'))
|
|
113
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_tenant_id'))
|
|
114
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_run_id'))
|
|
115
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_remote_id'))
|
|
116
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_entity_type'))
|
|
117
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_created_at'))
|
|
118
|
+
batch_op.drop_index(batch_op.f('ix_sync_record_log_action'))
|
|
119
|
+
|
|
120
|
+
op.drop_table('sync_record_log')
|
|
121
|
+
with op.batch_alter_table('sync_mapping', schema=None) as batch_op:
|
|
122
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_tenant_scope'))
|
|
123
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_tenant_id'))
|
|
124
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_status'))
|
|
125
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_remote_id'))
|
|
126
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_local_id'))
|
|
127
|
+
batch_op.drop_index(batch_op.f('ix_sync_mapping_entity_type'))
|
|
128
|
+
|
|
129
|
+
op.drop_table('sync_mapping')
|
|
130
|
+
# ### end Alembic commands ###
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Sync tables: the identity ledger, per-run aggregates, and an append-only record log.
|
|
2
|
+
|
|
3
|
+
A sync reconciles a local entity type against an external system. Three tables back it:
|
|
4
|
+
|
|
5
|
+
* :class:`SyncMapping` (``BaseTable``) — the identity ledger tying a local row to its remote
|
|
6
|
+
counterpart. Unique on (``tenant_scope``, ``entity_type``, ``local_id``) **and**
|
|
7
|
+
(``tenant_scope``, ``entity_type``, ``remote_id``), so an upsert is idempotent from either
|
|
8
|
+
side without colliding across tenants — the natural at-least-once dedupe key the design (§6
|
|
9
|
+
rule 3) leans on.
|
|
10
|
+
* :class:`SyncRun` (``BaseTable``) — one reconcile attempt's aggregates (counts + the
|
|
11
|
+
high-watermark ``cursor``), stored so a stats view never pays a per-row ``COUNT(*)``
|
|
12
|
+
(review M5).
|
|
13
|
+
* :class:`SyncRecordLog` — one append-only line per record processed, immutable exactly like
|
|
14
|
+
:class:`~terp.capabilities.audit.AuditEvent` (``UUIDPrimaryKeyMixin``, no OCC ``version`` /
|
|
15
|
+
``updated_at``); high-volume, so plan retention.
|
|
16
|
+
|
|
17
|
+
Every caller-influenceable ``str`` column caps its length.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import uuid
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
from typing import Final
|
|
25
|
+
|
|
26
|
+
from sqlalchemy import DateTime, UniqueConstraint
|
|
27
|
+
from sqlmodel import Field, SQLModel
|
|
28
|
+
|
|
29
|
+
from terp.core import BaseTable, UUIDPrimaryKeyMixin
|
|
30
|
+
|
|
31
|
+
# Mapping/run status + record-log actions (plain str columns, dependency-light leaves like
|
|
32
|
+
# ``AuditEvent.action`` — never a higher-layer enum on the table).
|
|
33
|
+
STATUS_SYNCED: Final[str] = "synced"
|
|
34
|
+
STATUS_RUNNING: Final[str] = "running"
|
|
35
|
+
STATUS_SUCCEEDED: Final[str] = "succeeded"
|
|
36
|
+
STATUS_FAILED: Final[str] = "failed"
|
|
37
|
+
|
|
38
|
+
ACTION_CREATED: Final[str] = "created"
|
|
39
|
+
ACTION_UPDATED: Final[str] = "updated"
|
|
40
|
+
ACTION_UNCHANGED: Final[str] = "unchanged"
|
|
41
|
+
ACTION_FAILED: Final[str] = "failed"
|
|
42
|
+
|
|
43
|
+
# Hard caps so a hostile / oversized value can never break the INSERT or the ledger.
|
|
44
|
+
_TYPE_MAX: Final[int] = 128
|
|
45
|
+
_REMOTE_ID_MAX: Final[int] = 200
|
|
46
|
+
_CHECKSUM_MAX: Final[int] = 128
|
|
47
|
+
_STATUS_MAX: Final[int] = 16
|
|
48
|
+
_ACTION_MAX: Final[int] = 16
|
|
49
|
+
_CURSOR_MAX: Final[int] = 512
|
|
50
|
+
_MESSAGE_MAX: Final[int] = 2000
|
|
51
|
+
_GLOBAL_TENANT_SCOPE: Final[str] = "global"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _utc_now() -> datetime:
|
|
55
|
+
"""UTC ``now`` provider for the timestamp columns."""
|
|
56
|
+
return datetime.now(UTC)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class SyncMapping(BaseTable, table=True):
|
|
60
|
+
"""The identity ledger: one local row ↔ one remote row for an ``entity_type``.
|
|
61
|
+
|
|
62
|
+
``id`` / ``created_at`` / ``updated_at`` / ``version`` are inherited from ``BaseTable``.
|
|
63
|
+
The two unique constraints make the mapping the idempotent upsert key from either side, so
|
|
64
|
+
an at-least-once redelivery of the same remote record never double-creates a local row.
|
|
65
|
+
``remote_checksum`` lets the reconcile detect a change without deep-diffing the payload.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
__tablename__ = "sync_mapping"
|
|
69
|
+
__table_args__ = (
|
|
70
|
+
UniqueConstraint(
|
|
71
|
+
"tenant_scope", "entity_type", "local_id", name="uq_sync_mapping_local"
|
|
72
|
+
),
|
|
73
|
+
UniqueConstraint(
|
|
74
|
+
"tenant_scope", "entity_type", "remote_id", name="uq_sync_mapping_remote"
|
|
75
|
+
),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
79
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
80
|
+
entity_type: str = Field(max_length=_TYPE_MAX, index=True)
|
|
81
|
+
local_id: uuid.UUID = Field(index=True)
|
|
82
|
+
remote_id: str = Field(max_length=_REMOTE_ID_MAX, index=True)
|
|
83
|
+
remote_checksum: str = Field(max_length=_CHECKSUM_MAX)
|
|
84
|
+
status: str = Field(default=STATUS_SYNCED, max_length=_STATUS_MAX, index=True)
|
|
85
|
+
last_synced_at: datetime = Field(
|
|
86
|
+
default_factory=_utc_now,
|
|
87
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
88
|
+
nullable=False,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class SyncRun(BaseTable, table=True):
|
|
93
|
+
"""One reconcile attempt for a ``source`` (entity type): status, counts, and cursor.
|
|
94
|
+
|
|
95
|
+
Aggregates are stored here (``processed_count`` / ``created_count`` / ``updated_count`` /
|
|
96
|
+
``failed_count``) so a stats view reads a single row instead of counting the log; the
|
|
97
|
+
high-watermark ``cursor`` is where the next run resumes.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
__tablename__ = "sync_run"
|
|
101
|
+
|
|
102
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
103
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
104
|
+
source: str = Field(max_length=_TYPE_MAX, index=True)
|
|
105
|
+
status: str = Field(default=STATUS_RUNNING, max_length=_STATUS_MAX, index=True)
|
|
106
|
+
started_at: datetime = Field(
|
|
107
|
+
default_factory=_utc_now,
|
|
108
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
109
|
+
nullable=False,
|
|
110
|
+
index=True,
|
|
111
|
+
)
|
|
112
|
+
finished_at: datetime | None = Field(
|
|
113
|
+
default=None,
|
|
114
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
115
|
+
nullable=True,
|
|
116
|
+
)
|
|
117
|
+
processed_count: int = Field(default=0)
|
|
118
|
+
created_count: int = Field(default=0)
|
|
119
|
+
updated_count: int = Field(default=0)
|
|
120
|
+
failed_count: int = Field(default=0)
|
|
121
|
+
cursor: str | None = Field(default=None, max_length=_CURSOR_MAX)
|
|
122
|
+
error: str | None = Field(default=None, max_length=_MESSAGE_MAX)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class SyncRecordLog(UUIDPrimaryKeyMixin, SQLModel, table=True): # arch-allow-table-models-use-base-table: append-only per-record log (like AuditEvent) — immutable, no version/updated_at by design (see module docstring)
|
|
126
|
+
__tablename__ = "sync_record_log"
|
|
127
|
+
|
|
128
|
+
run_id: uuid.UUID = Field(index=True)
|
|
129
|
+
tenant_scope: str = Field(default=_GLOBAL_TENANT_SCOPE, max_length=36, index=True)
|
|
130
|
+
tenant_id: uuid.UUID | None = Field(default=None, index=True)
|
|
131
|
+
entity_type: str = Field(max_length=_TYPE_MAX, index=True)
|
|
132
|
+
remote_id: str = Field(max_length=_REMOTE_ID_MAX, index=True)
|
|
133
|
+
action: str = Field(max_length=_ACTION_MAX, index=True)
|
|
134
|
+
message: str | None = Field(default=None, max_length=_MESSAGE_MAX)
|
|
135
|
+
created_at: datetime = Field(
|
|
136
|
+
default_factory=_utc_now,
|
|
137
|
+
sa_type=DateTime(timezone=True), # type: ignore[call-overload]
|
|
138
|
+
nullable=False,
|
|
139
|
+
index=True,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
__all__ = [
|
|
144
|
+
"ACTION_CREATED",
|
|
145
|
+
"ACTION_FAILED",
|
|
146
|
+
"ACTION_UNCHANGED",
|
|
147
|
+
"ACTION_UPDATED",
|
|
148
|
+
"STATUS_FAILED",
|
|
149
|
+
"STATUS_RUNNING",
|
|
150
|
+
"STATUS_SUCCEEDED",
|
|
151
|
+
"STATUS_SYNCED",
|
|
152
|
+
"SyncMapping",
|
|
153
|
+
"SyncRecordLog",
|
|
154
|
+
"SyncRun",
|
|
155
|
+
]
|
|
File without changes
|