pyarchinit-mini 2.9.4__tar.gz → 2.9.5__tar.gz

{pyarchinit_mini-2.9.4/pyarchinit_mini.egg-info → pyarchinit_mini-2.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyarchinit-mini
-Version: 2.9.4
+Version: 2.9.5
 Summary: Lightweight archaeological data management system with REST API, Web UI, Desktop GUI, CLI, multi-user authentication, real-time collaboration, analytics dashboard, PyArchInit import/export, Heriverse/ATON export integration, and 3D CRUD viewer
 Home-page: https://github.com/enzococca/pyarchinit-mini
 Author: PyArchInit Team

{pyarchinit_mini-2.9.4 → pyarchinit_mini-2.9.5}/pyarchinit_mini/__init__.py

@@ -23,7 +23,7 @@ Features:
 - Scalable and modular architecture
 """
 
-__version__ = "2.9.4"
+__version__ = "2.9.5"
 __author__ = "PyArchInit Team"
 __email__ = "enzo.ccc@gmail.com"
 

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/__init__.py

@@ -0,0 +1 @@
+"""One-way sync engine: classic pyarchinit -> pyarchinit_v2 (mini)."""

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/__main__.py

@@ -0,0 +1,22 @@
+import argparse, logging, sys
+from .config import load_config
+from .runner import run
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="python -m pyarchinit_mini.sync",
+                                description="Sync classic pyarchinit -> pyarchinit_v2")
+    p.add_argument("--config", default=None)
+    p.add_argument("--apply", action="store_true", help="write changes (default: dry-run)")
+    p.add_argument("--tables", default=None, help="comma-separated subset")
+    p.add_argument("--log", default=None, help="log file (default: stderr)")
+    args = p.parse_args(argv)
+    logging.basicConfig(level=logging.INFO, filename=args.log,
+                        format="%(asctime)s %(levelname)s %(message)s")
+    cfg = load_config(args.config)
+    tables = [t.strip() for t in args.tables.split(",")] if args.tables else None
+    results = run(cfg, tables=tables, dry_run=not args.apply)
+    errors = [r for r in results if r.error]
+    sys.exit(1 if errors else 0)
+
+if __name__ == "__main__":
+    main()

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/config.py

@@ -0,0 +1,45 @@
+import json, os
+from dataclasses import dataclass, field
+from typing import Mapping
+
+DEFAULT_PRESERVE = frozenset({
+    "order_layer", "cont_per", "entity_uuid", "node_uuid", "version_number",
+    "created_at", "updated_at", "last_modified_timestamp", "last_modified_by",
+    "editing_since", "editing_by", "audit_trail", "sync_status",
+})
+DEFAULT_EXCLUDE = frozenset({"spatial_ref_sys", "raster_columns", "raster_overviews"})
+
+@dataclass
+class Config:
+    source_dsn: str
+    target_dsn: str
+    size_threshold_keyset: int = 200_000
+    exclude_tables: frozenset[str] = DEFAULT_EXCLUDE
+    preserve_columns_global: frozenset[str] = DEFAULT_PRESERVE
+    overrides: dict[str, dict] = field(default_factory=dict)
+    weekly_full_refresh: bool = True
+    delete_enabled: bool = True
+    delete_on_empty_source: bool = False
+    collision_id_base: int = 1_000_000_000
+
+def load_config(path: str | None = None, env: Mapping[str, str] = os.environ) -> Config:
+    raw = {}
+    if path:
+        with open(path, encoding="utf-8") as fh:
+            raw = json.load(fh)
+    src_env = raw.get("source_dsn_env", "PYARCHINIT_CLASSIC_DSN")
+    tgt_env = raw.get("target_dsn_env", "DATABASE_URL")
+    source_dsn = env[src_env]          # KeyError if missing — intentional
+    target_dsn = env[tgt_env]
+    preserve = DEFAULT_PRESERVE | frozenset(raw.get("preserve_columns_global", []))
+    exclude = DEFAULT_EXCLUDE | frozenset(raw.get("exclude_tables", []))
+    return Config(
+        source_dsn=source_dsn, target_dsn=target_dsn,
+        size_threshold_keyset=int(raw.get("size_threshold_keyset", 200_000)),
+        exclude_tables=exclude, preserve_columns_global=preserve,
+        overrides=raw.get("overrides", {}),
+        weekly_full_refresh=bool(raw.get("weekly_full_refresh", True)),
+        delete_enabled=bool(raw.get("delete_enabled", True)),
+        delete_on_empty_source=bool(raw.get("delete_on_empty_source", False)),
+        collision_id_base=int(raw.get("collision_id_base", 1_000_000_000)),
+    )

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/diff.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+@dataclass
+class Diff:
+    inserts: list
+    updates: list
+    deletes: list
+
+def diff_by_hash(source: dict[tuple, str], target: dict[tuple, str]) -> Diff:
+    src_keys, tgt_keys = set(source), set(target)
+    inserts = sorted(src_keys - tgt_keys)
+    deletes = sorted(tgt_keys - src_keys)
+    updates = sorted(k for k in (src_keys & tgt_keys) if source[k] != target[k])
+    return Diff(inserts=inserts, updates=updates, deletes=deletes)
+
+def diff_by_keyset(source_keys: set[tuple], target_keys: set[tuple]) -> Diff:
+    return Diff(inserts=sorted(source_keys - target_keys),
+                updates=[],
+                deletes=sorted(target_keys - source_keys))

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/engine.py

@@ -0,0 +1,171 @@
+import re
+from dataclasses import dataclass
+from psycopg2.extensions import connection
+from . import introspect as I, transform as T, state as S, rowmap as M
+from .diff import diff_by_hash
+from .policy import select_mode, is_gated, preserve_set_for_table, common_data_columns
+
+@dataclass
+class TableResult:
+    table: str; mode: str; inserted: int; updated: int; deleted: int
+    skipped: bool; error: str | None
+
+_FILL_DEFAULTS = {
+    "created_at": "now()", "updated_at": "now()", "version_number": "1",
+    "entity_uuid": "gen_random_uuid()::text", "node_uuid": "gen_random_uuid()::text",
+}
+
+def _source_hash(src_conn, table, pk, common, src_types, tgt_types, geom):
+    hcols = [c for c in common if c != pk]      # pk excluded; identity is the map, not the hash
+    cur = src_conn.cursor()
+    cur.execute(T.build_pk_hash_select_coerced(table, [pk], hcols, src_types, tgt_types, geom))
+    return {str(r[0]): r[1] for r in cur.fetchall()}
+
+def _target_hash(tgt_conn, table, pk, common, tgt_types):
+    hcols = [c for c in common if c != pk]
+    cur = tgt_conn.cursor()
+    cur.execute(f'select "{pk}"::text, {T.plain_row_hash_sql(hcols, tgt_types)} from public."{table}"')
+    return {r[0]: r[1] for r in cur.fetchall()}
+
+def _fetch_source_row(src_conn, table, common, pk, v1_pk):
+    cur = src_conn.cursor()
+    cols = ", ".join(f'"{c}"' for c in common)
+    cur.execute(f'select {cols} from public."{table}" where "{pk}"::text=%s', (v1_pk,))
+    return cur.fetchone()
+
+def _value_exprs(common, src_types, tgt_types, geom):
+    out = []
+    for c in common:
+        tgt_t = "geometry" if c in geom else tgt_types[c][0]
+        out.append(T.cast_expr(src_types[c][0], tgt_t, tgt_types[c][1], ph=f"%({c})s"))
+    return out
+
+def _insert_one(tgt_conn, table, common, pk, row, v2_pk, src_types, tgt_types, geom):
+    # row is aligned to `common`; force the pk column to the chosen v2_pk
+    exprs = _value_exprs(common, src_types, tgt_types, geom)
+    fill = {c: v for c, v in _FILL_DEFAULTS.items() if c in tgt_types and c not in common}
+    cols = common + list(fill.keys())
+    col_sql = ", ".join(f'"{c}"' for c in cols)
+    val_sql = ", ".join(exprs + list(fill.values()))
+    params = {c: row[i] for i, c in enumerate(common)}
+    params[pk] = v2_pk                       # override pk value with the chosen v2_pk
+    tgt_conn.cursor().execute(
+        f'INSERT INTO public."{table}" ({col_sql}) VALUES ({val_sql})', params)
+
+def _update_one(tgt_conn, table, common, pk, row, v2_pk, src_types, tgt_types, geom):
+    set_cols = [c for c in common if c != pk]            # never update the pk
+    exprs = _value_exprs(set_cols, src_types, tgt_types, geom)
+    assigns = ", ".join(f'"{c}" = {e}' for c, e in zip(set_cols, exprs))
+    params = {c: row[common.index(c)] for c in set_cols}
+    params["__v2pk"] = v2_pk
+    tgt_conn.cursor().execute(
+        f'UPDATE public."{table}" SET {assigns} WHERE "{pk}"::text = %(__v2pk)s', params)
+
+def _delete_one(tgt_conn, table, pk, v2_pk):
+    tgt_conn.cursor().execute(f'delete from public."{table}" where "{pk}"::text=%s', (v2_pk,))
+
+def _alloc_v2_pk(v1_pk, v2pks, state):
+    if v1_pk not in v2pks:
+        return v1_pk
+    nxt = state["next_high"]
+    while str(nxt) in v2pks:
+        nxt += 1
+    state["next_high"] = nxt + 1
+    return str(nxt)
+
+def sync_table(src_conn, tgt_conn, table, cfg, dry_run=True) -> TableResult:
+    mode = "unknown"; ins = upd = dele = 0
+    try:
+        M.ensure_map_table(tgt_conn); S.ensure_state_table(tgt_conn)
+        pk_cols = I.primary_key(src_conn, table)
+        single_pk = pk_cols[0] if len(pk_cols) == 1 else None
+        mode = select_mode(single_pk is not None)
+        rc = I.row_count(src_conn, table)
+        src_types = I.column_types(src_conn, table)
+        tgt_types = I.column_types(tgt_conn, table)
+        geom = I.geometry_columns(tgt_conn, table)
+        preserve = preserve_set_for_table(cfg.preserve_columns_global,
+                                          set(src_types), set(tgt_types), [])
+        common = common_data_columns(set(src_types), set(tgt_types), preserve)
+        if mode == "additive":
+            src_hexpr = T.coerced_row_hash_sql(common, src_types, tgt_types, geom)
+            tgt_hexpr = T.plain_row_hash_sql(common, tgt_types)
+            scur = src_conn.cursor()
+            scur.execute(f'select {src_hexpr}, {", ".join(chr(34)+c+chr(34) for c in common)} '
+                         f'from public."{table}"')
+            srows = scur.fetchall()
+            tcur = tgt_conn.cursor()
+            tcur.execute(f'select {tgt_hexpr} from public."{table}"')
+            seen = {r[0] for r in tcur.fetchall()}
+            exprs = _value_exprs(common, src_types, tgt_types, geom)
+            col_sql = ", ".join(f'"{c}"' for c in common)
+            sql = f'INSERT INTO public."{table}" ({col_sql}) VALUES ({", ".join(exprs)})'
+            icur = tgt_conn.cursor()
+            for r in srows:
+                h = r[0]
+                if h in seen:
+                    continue
+                seen.add(h)
+                row = r[1:]
+                params = {c: row[i] for i, c in enumerate(common)}
+                icur.execute(sql, params)
+                ins += 1
+            if dry_run:
+                tgt_conn.rollback()
+            else:
+                S.record_result(tgt_conn, table, I.signature(src_conn, table, pk_cols),
+                                mode, ins, 0, 0, None)
+                tgt_conn.commit()
+            return TableResult(table, mode, ins, 0, 0, False, None)
+        if is_gated(rc, cfg.size_threshold_keyset):
+            if I.signature(src_conn, table, pk_cols) == S.get_signature(tgt_conn, table):
+                tgt_conn.rollback()
+                return TableResult(table, mode, 0, 0, 0, True, None)
+        if M.map_count(tgt_conn, table) == 0:
+            M.bootstrap_table(tgt_conn, src_conn, table, single_pk)
+        mp = M.load_map(tgt_conn, table)                       # {v1_pk: v2_pk}
+        src = _source_hash(src_conn, table, single_pk, common, src_types, tgt_types, geom)
+        tgt_h = _target_hash(tgt_conn, table, single_pk, common, tgt_types)   # {v2_pk: hash}
+        target = {v1: tgt_h.get(v2) for v1, v2 in mp.items()}      # {v1_pk: hash-of-its-v2-row}
+        d = diff_by_hash(src, target)
+        v2pks = M.v2_pk_set(tgt_conn, table, single_pk)
+        nums = [int(x) for x in v2pks if re.fullmatch(r"-?\d+", x)]
+        state = {"next_high": max([cfg.collision_id_base] + [n + 1 for n in nums])}
+        if cfg.delete_enabled and (rc > 0 or cfg.delete_on_empty_source):
+            for v1k in d.deletes:
+                _delete_one(tgt_conn, table, single_pk, mp[v1k]); M.delete_map(tgt_conn, table, v1k); dele += 1
+        for v1k in d.inserts:
+            row = _fetch_source_row(src_conn, table, common, single_pk, v1k)
+            if row is None:
+                continue
+            v2k = _alloc_v2_pk(v1k, v2pks, state)
+            _insert_one(tgt_conn, table, common, single_pk, row, v2k, src_types, tgt_types, geom)
+            M.upsert_map(tgt_conn, table, v1k, v2k); v2pks.add(v2k); ins += 1
+        for v1k in d.updates:
+            v2k = mp[v1k]
+            row = _fetch_source_row(src_conn, table, common, single_pk, v1k)
+            if row is None:
+                continue
+            if v2k in v2pks:
+                _update_one(tgt_conn, table, common, single_pk, row, v2k, src_types, tgt_types, geom)
+            else:                                   # mapped row deleted in v2 -> re-insert
+                v2k = _alloc_v2_pk(v1k, v2pks, state)
+                _insert_one(tgt_conn, table, common, single_pk, row, v2k, src_types, tgt_types, geom)
+                M.upsert_map(tgt_conn, table, v1k, v2k); v2pks.add(v2k)
+            upd += 1
+        if dry_run:
+            tgt_conn.rollback()
+        else:
+            S.record_result(tgt_conn, table, I.signature(src_conn, table, pk_cols),
+                            mode, ins, upd, dele, None)
+            tgt_conn.commit()
+        return TableResult(table, mode, ins, upd, dele, False, None)
+    except Exception as e:
+        tgt_conn.rollback()
+        msg = str(e).splitlines()[0]
+        try:
+            S.ensure_state_table(tgt_conn)
+            S.record_result(tgt_conn, table, "", mode, 0, 0, 0, msg); tgt_conn.commit()
+        except Exception:
+            tgt_conn.rollback()
+        return TableResult(table, mode, 0, 0, 0, False, msg)

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/introspect.py

@@ -0,0 +1,47 @@
+from psycopg2.extensions import connection
+
+
+def base_tables(conn: connection) -> set[str]:
+    cur = conn.cursor()
+    cur.execute("select table_name from information_schema.tables "
+                "where table_schema='public' and table_type='BASE TABLE'")
+    return {r[0] for r in cur.fetchall()}
+
+def column_types(conn: connection, table: str) -> dict[str, tuple[str, int | None]]:
+    cur = conn.cursor()
+    cur.execute("select column_name, data_type, character_maximum_length "
+                "from information_schema.columns where table_schema='public' and table_name=%s",
+                (table,))
+    return {r[0]: (r[1], r[2]) for r in cur.fetchall()}
+
+def primary_key(conn: connection, table: str) -> list[str]:
+    cur = conn.cursor()
+    cur.execute("""select a.attname from pg_index i
+                   join pg_class c on c.oid=i.indrelid
+                   join pg_namespace n on n.oid=c.relnamespace
+                   join pg_attribute a on a.attrelid=c.oid and a.attnum=any(i.indkey)
+                   where n.nspname='public' and c.relname=%s and i.indisprimary
+                   order by a.attnum""", (table,))
+    return [r[0] for r in cur.fetchall()]
+
+def geometry_columns(conn: connection, table: str) -> set[str]:
+    cur = conn.cursor()
+    cur.execute("select column_name from information_schema.columns "
+                "where table_schema='public' and table_name=%s "
+                "and udt_name in ('geometry','geography')", (table,))
+    return {r[0] for r in cur.fetchall()}
+
+def row_count(conn: connection, table: str) -> int:
+    cur = conn.cursor()
+    cur.execute(f'select count(*) from public."{table}"')
+    return cur.fetchone()[0]
+
+def signature(conn: connection, table: str, pk: list[str]) -> str:
+    cur = conn.cursor()
+    cur.execute(f'select count(*) from public."{table}"')
+    cnt = cur.fetchone()[0]
+    maxpk = ""
+    if pk:
+        cur.execute(f'select max("{pk[0]}") from public."{table}"')
+        maxpk = str(cur.fetchone()[0])
+    return f"{cnt}:{maxpk}"

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/policy.py

@@ -0,0 +1,13 @@
+def select_mode(has_single_pk: bool) -> str:
+    return "mapped" if has_single_pk else "additive"
+
+def is_gated(rowcount: int, threshold: int) -> bool:
+    return rowcount > threshold
+
+def preserve_set_for_table(global_preserve: frozenset[str], src_cols: set[str], tgt_cols: set[str], extra: list[str]) -> set[str]:
+    target_only = set(tgt_cols) - set(src_cols)
+    en_cols = {c for c in tgt_cols if c.endswith("_en")}
+    return set(global_preserve) | target_only | en_cols | set(extra or [])
+
+def common_data_columns(src_cols: set[str], tgt_cols: set[str], preserve: set[str]) -> list[str]:
+    return sorted((set(src_cols) & set(tgt_cols)) - set(preserve))

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/rowmap.py

@@ -0,0 +1,62 @@
+from psycopg2.extensions import connection
+from psycopg2.extras import execute_values
+
+
+def ensure_map_table(conn: connection) -> None:
+    conn.cursor().execute("""
+        CREATE TABLE IF NOT EXISTS public.sync_row_map (
+            table_name text NOT NULL,
+            v1_pk text NOT NULL,
+            v2_pk text NOT NULL,
+            last_run_at timestamptz DEFAULT now(),
+            PRIMARY KEY (table_name, v1_pk),
+            UNIQUE (table_name, v2_pk)
+        )""")
+
+
+def map_count(conn: connection, table: str) -> int:
+    cur = conn.cursor()
+    cur.execute("select count(*) from public.sync_row_map where table_name=%s", (table,))
+    return cur.fetchone()[0]
+
+
+def load_map(conn: connection, table: str) -> dict[str, str]:
+    cur = conn.cursor()
+    cur.execute("select v1_pk, v2_pk from public.sync_row_map where table_name=%s", (table,))
+    return {r[0]: r[1] for r in cur.fetchall()}
+
+
+def v2_pk_set(conn: connection, table: str, pk: str) -> set[str]:
+    cur = conn.cursor()
+    cur.execute(f'select "{pk}"::text from public."{table}"')
+    return {r[0] for r in cur.fetchall()}
+
+
+def bootstrap_table(tgt_conn: connection, src_conn: connection, table: str, pk: str) -> int:
+    if map_count(tgt_conn, table) > 0:      # only bootstrap an empty map
+        return 0
+    scur = src_conn.cursor()
+    scur.execute(f'select "{pk}"::text from public."{table}"')
+    v1 = {r[0] for r in scur.fetchall()}
+    v2 = v2_pk_set(tgt_conn, table, pk)
+    overlap = sorted(v1 & v2)
+    if not overlap:
+        return 0
+    tcur = tgt_conn.cursor()
+    execute_values(tcur,
+        "INSERT INTO public.sync_row_map (table_name, v1_pk, v2_pk) VALUES %s "
+        "ON CONFLICT (table_name, v1_pk) DO NOTHING",
+        [(table, k, k) for k in overlap])
+    return len(overlap)
+
+
+def upsert_map(conn: connection, table: str, v1_pk: str, v2_pk: str) -> None:
+    conn.cursor().execute(
+        "INSERT INTO public.sync_row_map (table_name, v1_pk, v2_pk) VALUES (%s,%s,%s) "
+        "ON CONFLICT (table_name, v1_pk) DO UPDATE SET v2_pk=EXCLUDED.v2_pk, last_run_at=now()",
+        (table, v1_pk, v2_pk))
+
+
+def delete_map(conn: connection, table: str, v1_pk: str) -> None:
+    conn.cursor().execute(
+        "delete from public.sync_row_map where table_name=%s and v1_pk=%s", (table, v1_pk))

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/runner.py

@@ -0,0 +1,33 @@
+import logging, psycopg2
+from psycopg2.extensions import connection
+from . import introspect as I
+from .config import Config
+from .engine import sync_table, TableResult
+
+def discover_tables(src_conn: connection, tgt_conn: connection, cfg: Config) -> list[str]:
+    mirrored = I.base_tables(src_conn) & I.base_tables(tgt_conn)
+    return sorted(mirrored - set(cfg.exclude_tables))
+
+def run(cfg: Config, tables: list[str] | None = None, dry_run: bool = True, logger=None, _conns: tuple | None = None) -> list[TableResult]:
+    logger = logger or logging.getLogger("sync")
+    src, tgt = _conns or (psycopg2.connect(cfg.source_dsn), psycopg2.connect(cfg.target_dsn))
+    src.autocommit = True              # source: read-only
+    owns = _conns is None
+    try:
+        names = tables if tables else discover_tables(src, tgt, cfg)
+        results = []
+        ti = tu = td = 0
+        for name in names:
+            r = sync_table(src, tgt, name, cfg, dry_run=dry_run)
+            results.append(r)
+            ti += r.inserted; tu += r.updated; td += r.deleted
+            tag = "DRY" if dry_run else "APPLY"
+            state = "SKIP" if r.skipped else ("ERR:" + r.error if r.error else "ok")
+            logger.info("[%s] %-40s mode=%-7s +%d ~%d -%d %s",
+                        tag, name, r.mode, r.inserted, r.updated, r.deleted, state)
+        logger.info("[%s] TOTAL tables=%d +%d ~%d -%d",
+                    "DRY" if dry_run else "APPLY", len(results), ti, tu, td)
+        return results
+    finally:
+        if owns:
+            src.close(); tgt.close()

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/state.py

@@ -0,0 +1,44 @@
+"""sync_state tracking table — records last sync result per table."""
+from psycopg2.extensions import connection
+
+
+def ensure_state_table(conn: connection) -> None:
+    conn.cursor().execute("""
+        CREATE TABLE IF NOT EXISTS public.sync_state (
+            table_name text PRIMARY KEY,
+            last_signature text,
+            last_run_at timestamptz DEFAULT now(),
+            last_mode text,
+            rows_inserted int DEFAULT 0,
+            rows_updated int DEFAULT 0,
+            rows_deleted int DEFAULT 0,
+            error text
+        )""")
+
+
+def get_signature(conn: connection, table: str) -> str | None:
+    cur = conn.cursor()
+    cur.execute("select last_signature from public.sync_state where table_name=%s", (table,))
+    row = cur.fetchone()
+    return row[0] if row else None
+
+
+def record_result(
+    conn: connection,
+    table: str,
+    signature: str,
+    mode: str,
+    inserted: int,
+    updated: int,
+    deleted: int,
+    error: str | None,
+) -> None:
+    conn.cursor().execute("""
+        INSERT INTO public.sync_state
+          (table_name, last_signature, last_run_at, last_mode, rows_inserted, rows_updated, rows_deleted, error)
+        VALUES (%s,%s,now(),%s,%s,%s,%s,%s)
+        ON CONFLICT (table_name) DO UPDATE SET
+          last_signature=EXCLUDED.last_signature, last_run_at=now(), last_mode=EXCLUDED.last_mode,
+          rows_inserted=EXCLUDED.rows_inserted, rows_updated=EXCLUDED.rows_updated,
+          rows_deleted=EXCLUDED.rows_deleted, error=EXCLUDED.error
+        """, (table, signature, mode, inserted, updated, deleted, error))

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/sync_config.example.json

@@ -0,0 +1,10 @@
+{
+  "source_dsn_env": "PYARCHINIT_CLASSIC_DSN",
+  "target_dsn_env": "DATABASE_URL",
+  "size_threshold_keyset": 200000,
+  "exclude_tables": [],
+  "preserve_columns_global": [],
+  "overrides": {},
+  "weekly_full_refresh": true,
+  "delete_enabled": true
+}

pyarchinit_mini-2.9.5/pyarchinit_mini/sync/transform.py

@@ -0,0 +1,90 @@
+def cast_expr(src_type: str, tgt_type: str, tgt_maxlen: int | None, ph: str = "%s") -> str:
+    # CHAR(n) (bpchar) source is blank-padded; trailing spaces are insignificant.
+    # Strip them so the coerced value matches the trimmed value stored in v2.
+    if src_type == "character":
+        ph = f"rtrim({ph})"
+    t = tgt_type
+    if t in ("geometry", "geography"):
+        return f"({ph})::{t}"
+    if t == "character varying" and tgt_maxlen:
+        return f"left(({ph})::text, {tgt_maxlen})"
+    if t == "text":
+        return f"({ph})::text"
+    if t in ("integer", "bigint", "smallint"):
+        if src_type in ("character varying", "text"):
+            return (f"(CASE WHEN btrim(({ph})::text) ~ '^-?[0-9]+$' "
+                    f"THEN btrim(({ph})::text)::{t} ELSE NULL END)")
+        return f"({ph})::{t}"
+    if t == "boolean":
+        if src_type == "boolean":
+            return f"({ph})"
+        if src_type in ("bigint", "integer", "smallint", "numeric", "double precision"):
+            return (f"(CASE WHEN {ph} IS NULL THEN NULL "
+                    f"WHEN ({ph})::numeric=0 THEN false ELSE true END)")
+        return (f"(CASE WHEN btrim(({ph})::text) IN ('1','true','t','yes','si','y') THEN true "
+                f"WHEN btrim(({ph})::text) IN ('0','false','f','no','n','') THEN false ELSE NULL END)")
+    if t == "date":
+        if src_type == "date":
+            return f"({ph})"
+        return ("(CASE "
+                f"WHEN btrim(({ph})::text) ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}' THEN substr(btrim(({ph})::text),1,10)::date "
+                f"WHEN btrim(({ph})::text) ~ '^[0-9]{{1,2}}/[0-9]{{1,2}}/[0-9]{{4}}$' THEN to_date(btrim(({ph})::text),'DD/MM/YYYY') "
+                f"WHEN btrim(({ph})::text) ~ '^[0-9]{{1,2}}-[0-9]{{1,2}}-[0-9]{{4}}$' THEN to_date(btrim(({ph})::text),'DD-MM-YYYY') "
+                "ELSE NULL END)")
+    if t == "timestamp with time zone":
+        return f"({ph})::timestamptz"
+    if t == "timestamp without time zone":
+        return f"({ph})::timestamp"
+    if t == "double precision":
+        return f"({ph})::double precision"
+    if t == "numeric":
+        return f"({ph})::numeric"
+    return f"({ph})"   # matching/other types: pass the value through
+
+def row_hash_sql(columns: list[str]) -> str:
+    parts = "||'|'||".join(f"coalesce(\"{c}\"::text,'')" for c in columns)
+    return f"md5({parts})" if columns else "md5('')"
+
+def plain_row_hash_sql(columns: list[str], col_types: dict) -> str:
+    if not columns:
+        return "md5('')"
+    parts = []
+    for c in columns:
+        ref = f'"{c}"'
+        if col_types.get(c, ("", None))[0] == "character":
+            ref = f"rtrim({ref})"
+        parts.append(f"coalesce(({ref})::text,'')")
+    return "md5(" + "||'|'||".join(parts) + ")"
+
+def build_pk_hash_select(table: str, pk: list[str], hash_cols: list[str]) -> str:
+    pk_sql = ", ".join(f'"{c}"' for c in pk)
+    return f'SELECT {pk_sql}, {row_hash_sql(hash_cols)} FROM public."{table}"'
+
+def coerced_row_hash_sql(columns: list[str], src_types: dict, tgt_types: dict, geom: set) -> str:
+    if not columns:
+        return "md5('')"
+    parts = []
+    for c in columns:
+        tgt_t = "geometry" if c in geom else tgt_types[c][0]
+        expr = cast_expr(src_types[c][0], tgt_t, tgt_types[c][1], ph=f'"{c}"')
+        parts.append(f"coalesce(({expr})::text,'')")
+    return "md5(" + "||'|'||".join(parts) + ")"
+
+def build_pk_hash_select_coerced(table: str, pk: list[str], columns: list[str],
+                                 src_types: dict, tgt_types: dict, geom: set) -> str:
+    pk_sql = ", ".join(f'"{c}"' for c in pk)
+    return (f'SELECT {pk_sql}, '
+            f'{coerced_row_hash_sql(columns, src_types, tgt_types, geom)} FROM public."{table}"')
+
+def build_insert(table: str, cols: list[str], value_exprs: list[str], fill: dict[str, str]) -> str:
+    all_cols = cols + list(fill.keys())
+    col_sql = ", ".join(f'"{c}"' for c in all_cols)
+    val_sql = ", ".join(value_exprs + list(fill.values()))
+    return f'INSERT INTO public."{table}" ({col_sql}) VALUES ({val_sql})'
+
+def build_update(table: str, set_cols: list[str], set_exprs: list[str], pk: list[str]) -> str:
+    # pk uses NAMED placeholders so the whole statement is named-param (set_exprs
+    # carry named %(col)s placeholders from cast_expr; mixing named+positional is illegal).
+    assigns = ", ".join(f'"{c}" = {e}' for c, e in zip(set_cols, set_exprs))
+    where = " AND ".join(f'"{c}" = %(__pk_{c})s' for c in pk)
+    return f'UPDATE public."{table}" SET {assigns} WHERE {where}'

{pyarchinit_mini-2.9.4 → pyarchinit_mini-2.9.5/pyarchinit_mini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyarchinit-mini
-Version: 2.9.4
+Version: 2.9.5
 Summary: Lightweight archaeological data management system with REST API, Web UI, Desktop GUI, CLI, multi-user authentication, real-time collaboration, analytics dashboard, PyArchInit import/export, Heriverse/ATON export integration, and 3D CRUD viewer
 Home-page: https://github.com/enzococca/pyarchinit-mini
 Author: PyArchInit Team

{pyarchinit_mini-2.9.4 → pyarchinit_mini-2.9.5}/pyarchinit_mini.egg-info/SOURCES.txt

@@ -275,7 +275,21 @@ pyarchinit_mini/stratigraph/sync_orchestrator.py
 pyarchinit_mini/stratigraph/sync_queue.py
 pyarchinit_mini/stratigraph/sync_state_machine.py
 pyarchinit_mini/stratigraph/uuid_manager.py
+pyarchinit_mini/sync/__init__.py
+pyarchinit_mini/sync/__main__.py
+pyarchinit_mini/sync/config.py
+pyarchinit_mini/sync/diff.py
+pyarchinit_mini/sync/engine.py
+pyarchinit_mini/sync/introspect.py
+pyarchinit_mini/sync/policy.py
+pyarchinit_mini/sync/rowmap.py
+pyarchinit_mini/sync/runner.py
+pyarchinit_mini/sync/state.py
+pyarchinit_mini/sync/sync_config.example.json
+pyarchinit_mini/sync/transform.py
+pyarchinit_mini/translations/en/LC_MESSAGES/messages.mo
 pyarchinit_mini/translations/en/LC_MESSAGES/messages.po
+pyarchinit_mini/translations/it/LC_MESSAGES/messages.mo
 pyarchinit_mini/translations/it/LC_MESSAGES/messages.po
 pyarchinit_mini/utils/__init__.py
 pyarchinit_mini/utils/auth.py
@@ -451,6 +465,16 @@ tests/integration/test_vocab_routes.py
 tests/integration/test_yed_export_route.py
 tests/integration/test_yed_import_routes.py
 tests/integration/test_yed_writer_parity.py
+tests/sync/__init__.py
+tests/sync/conftest.py
+tests/sync/test_config.py
+tests/sync/test_diff.py
+tests/sync/test_engine.py
+tests/sync/test_introspect.py
+tests/sync/test_policy.py
+tests/sync/test_rowmap.py
+tests/sync/test_runner.py
+tests/sync/test_transform.py
 tests/unit/__init__.py
 tests/unit/test_ai_matrix_apply.py
 tests/unit/test_ai_matrix_extractor_parse.py

{pyarchinit_mini-2.9.4 → pyarchinit_mini-2.9.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyarchinit-mini"
-version = "2.9.4"
+version = "2.9.5"
 description = "Lightweight archaeological data management system with REST API, Web UI, Desktop GUI, CLI, multi-user authentication, real-time collaboration, analytics dashboard, PyArchInit import/export, Heriverse/ATON export integration, and 3D CRUD viewer"
 readme = "README.md"
 license = {text = "GPL-2.0"}
@@ -174,6 +174,7 @@ exclude = ["docs*", "examples*", "scripts*", "test_*"]
 "*" = ["logo/*.png"]
 pyarchinit_mini = [
     "*.yaml", "*.yml", "*.json",
+    "sync/*.json",
     "data/*.db", "data/*.sql",
     "graphml_converter/templates/*.graphml",
     "web_interface/templates/*.html",