sqlmirror 0.2.0__py3-none-any.whl

sqlmirror/dev/cli.py

@@ -0,0 +1,166 @@
+"""sqlmirror-dev: test tooling (synthetic tenants, mock Jira, live-site loaders, golden queries)."""
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import click
+from dotenv import load_dotenv
+
+from ..cli import _env, _jira
+
+
+@click.group()
+@click.option("--env-file", type=click.Path(dir_okay=False))
+def main(env_file: str | None) -> None:
+    """Development and test tools for SQL Mirror."""
+    load_dotenv(env_file or Path.cwd() / ".env", override=False)
+
+
+@main.command()
+@click.option("--issues", default=2000, show_default=True)
+@click.option("--projects", default=3, show_default=True)
+@click.option("--seed", default=7, show_default=True)
+@click.option("--out", default="data/tenant.json.gz", show_default=True)
+def synth(issues: int, projects: int, seed: int, out: str) -> None:
+    """Generate a synthetic Jira Cloud tenant (for the mock server)."""
+    from .synth import build_tenant
+
+    t = build_tenant(issues=issues, projects=projects, seed=seed)
+    t.save(out)
+    click.echo(f"Wrote {len(t.issues)} issues to {out}")
+
+
+@main.command("mock-jira")
+@click.option("--data", help="Tenant file from 'synth'. Default: generate --issues on the fly.")
+@click.option("--issues", default=2000, show_default=True)
+@click.option("--port", default=8899, show_default=True)
+@click.option("--flaky", default=0.0, help="Fraction of requests answered with 429.")
+def mock_jira(data: str | None, issues: int, port: int, flaky: float) -> None:
+    """Serve a synthetic tenant over the Jira REST API (for backfill and app tests)."""
+    from .mock_jira import serve
+    from .synth import Tenant, build_tenant
+
+    tenant = Tenant.load(data) if data else build_tenant(issues=issues)
+    click.echo(f"Mock Jira with {len(tenant.issues)} issues on http://127.0.0.1:{port} (Ctrl+C to stop)")
+    serve(tenant, port=port, flaky=flaky)
+
+
+@main.command()
+@click.option("--server-dsn", default=lambda: _env("SQLMIRROR_TEST_SERVER_DSN") or "postgresql://postgres@localhost:5433/postgres",
+              show_default="$SQLMIRROR_TEST_SERVER_DSN or postgresql://postgres@localhost:5433/postgres",
+              help="Postgres server to create the sqlm_e2e_* test databases on.")
+@click.option("--forge-dir", default=None, help="Path to the Forge app (default: ../sql-mirror next to cli/).")
+@click.option("--issues", default=600, show_default=True)
+@click.option("--mssql-dsn", default=lambda: _env("SQLMIRROR_TEST_MSSQL_DSN"),
+              help="Also test a SQL Server mirror: server to create sqlm_e2e_* databases on, e.g. "
+                   "sqlserver://sa:password@localhost:1433/master (default $SQLMIRROR_TEST_MSSQL_DSN).")
+def e2e(server_dsn: str, forge_dir: str | None, issues: int, mssql_dsn: str | None) -> None:
+    """Offline end-to-end test: mock Jira + Forge app code + local S3 + Postgres (+ SQL Server)."""
+    import logging
+
+    from .e2e import run_e2e
+
+    logging.getLogger("werkzeug").setLevel(logging.ERROR)
+    forge_dir = forge_dir or str(Path(__file__).resolve().parents[3] / "sql-mirror")
+    ok = run_e2e(server_dsn, forge_dir, issues=issues, mssql_server_dsn=mssql_dsn)
+    sys.exit(0 if ok else 1)
+
+
+@main.command()
+@click.option("--dsn", default=lambda: _env("DATABASE_URL"), help="Mirror database (default $DATABASE_URL).")
+@click.option("--strict", is_flag=True, help="Treat 'no data of this kind' as a failure (for synthetic data).")
+@click.option("--compare-with", help="Another mirror of the same data (e.g. Postgres for a SQL Server mirror): "
+                                     "check both give the same answers.")
+def golden(dsn: str | None, strict: bool, compare_with: str | None) -> None:
+    """Run the 30 golden Data Center report queries against a mirror (Postgres or T-SQL set to match)."""
+    from .golden import compare_golden, run_golden
+
+    if not dsn:
+        raise click.UsageError("Set DATABASE_URL or pass --dsn")
+    results = run_golden(dsn, strict=strict)
+    for r in results:
+        click.echo(f"{r['status'].upper():<5} {r['name']:<32} {r['rows']:>5} rows {r['ms']:>5} ms  {r['error'] or ''}")
+    counts = {s: sum(1 for r in results if r["status"] == s) for s in ("pass", "empty", "fail")}
+    click.echo(f"{counts['pass']} passed, {counts['empty']} empty (site has no such data yet), {counts['fail']} failed")
+    differing = []
+    if compare_with:
+        verdicts = compare_golden(results, run_golden(compare_with, strict=strict))
+        differing = [v for v in verdicts if v["verdict"] != "same"]
+        click.echo(f"\nSame answers as {compare_with.split('@')[-1]}: {len(verdicts) - len(differing)}/{len(verdicts)}")
+        for v in differing:
+            click.echo(f"  {v['name']}: {v['verdict']}: {v['detail']}")
+    sys.exit(1 if counts["fail"] or differing else 0)
+
+
+@main.command()
+@click.argument("dsn_a")
+@click.argument("dsn_b")
+def compare(dsn_a: str, dsn_b: str) -> None:
+    """Compare two mirror databases table by table (Postgres and/or SQL Server)."""
+    from .compare import compare as run, format_report, identical
+
+    report = run(dsn_a, dsn_b)
+    click.echo(format_report(report))
+    sys.exit(0 if identical(report) else 1)
+
+
+def _demo_guard(yes: bool) -> None:
+    site = _env("JIRA_SITE") or ""
+    if not yes:
+        click.confirm(f"This creates and edits issues on {site}. Only use a demo/test site. Continue?", abort=True)
+
+
+@main.command("load-site")
+@click.option("--project", default="SQLM", show_default=True, help="Project key to create or fill.")
+@click.option("--issues", default=500, show_default=True)
+@click.option("--seed", default=1, show_default=True)
+@click.option("--yes", is_flag=True, help="Don't ask for confirmation.")
+def load_site_cmd(project: str, issues: int, seed: int, yes: bool) -> None:
+    """Create realistic test data on a demo site (uses JIRA_* settings)."""
+    from .site import load_site
+
+    _demo_guard(yes)
+    load_site(_jira(), project=project, issues=issues, seed=seed)
+
+
+@main.command("enrich-site")
+@click.option("--project", default="SQLM", show_default=True)
+@click.option("--yes", is_flag=True)
+def enrich_site_cmd(project: str, yes: bool) -> None:
+    """Add sprints, attachments, estimates, reopened issues etc. to existing demo issues."""
+    from .site import enrich_site
+
+    _demo_guard(yes)
+    enrich_site(_jira(), project=project)
+
+
+@main.command("simulate-edits")
+@click.option("--project", default="SQLM", show_default=True)
+@click.option("--per-hour", default=300, show_default=True, help="Edits per hour (300 is a busy 250-user team).")
+@click.option("--minutes", default=60, show_default=True)
+@click.option("--yes", is_flag=True)
+def simulate_edits_cmd(project: str, per_hour: int, minutes: int, yes: bool) -> None:
+    """Make a steady stream of edits on a demo site (drives the app's events)."""
+    from .site import simulate_edits
+
+    _demo_guard(yes)
+    simulate_edits(_jira(), project=project, per_hour=per_hour, minutes=minutes)
+
+
+@main.command("bulk-edit")
+@click.option("--project", default="SQLM", show_default=True)
+@click.option("--count", default=5000, show_default=True)
+@click.option("--workers", default=8, show_default=True)
+@click.option("--yes", is_flag=True)
+def bulk_edit_cmd(project: str, count: int, workers: int, yes: bool) -> None:
+    """Edit many issues at once (the bulk-edit rate-limit test)."""
+    from .site import bulk_edit
+
+    _demo_guard(yes)
+    bulk_edit(_jira(), project=project, count=count, workers=workers)
+
+
+if __name__ == "__main__":
+    main()

sqlmirror/dev/compare.py

@@ -0,0 +1,87 @@
+"""Compare two mirror databases table by table (used to prove two sync paths agree).
+
+Works across databases too: a SQL Server mirror against a Postgres mirror of
+the same data. Then timestamps are compared at SQL Server's datetime
+resolution (1/300 s), since that is all datetime keeps.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from ..schema import TABLES
+
+
+def _open(target: Any) -> tuple[Any, bool]:
+    if isinstance(target, str):
+        from ..db import open_mirror
+
+        return open_mirror(target), True
+    return target, False
+
+
+def _norm_cross(db: Any, v: Any) -> Any:
+    if isinstance(v, datetime):
+        if v.tzinfo is None:  # SQL Server: naive, in the mirror's zone
+            v = db.local_to_aware(v) if hasattr(db, "local_to_aware") else v.replace(tzinfo=timezone.utc)
+        v = v.astimezone(timezone.utc).replace(tzinfo=None)
+        return ("ts", round((v - datetime(1900, 1, 1)).total_seconds() * 300))
+    return v
+
+
+def _rows(db: Any, table, cross: bool) -> dict[tuple, tuple]:
+    cols = table.column_names
+    pk_idx = [cols.index(c) for c in table.pk]
+    out = {}
+    for r in db.read_table(table):
+        r = tuple(_norm_cross(db, v) for v in r) if cross else tuple(r)
+        out[tuple(r[i] for i in pk_idx)] = r
+    return out
+
+
+def compare(a: Any, b: Any, sample: int = 3) -> list[dict[str, Any]]:
+    """Return one entry per table with counts of rows only in A, only in B, and different.
+
+    ``a`` and ``b`` are connection strings or open mirrors.
+    """
+    report = []
+    ma, close_a = _open(a)
+    mb, close_b = _open(b)
+    cross = ma.dialect != mb.dialect
+    try:
+        for t in TABLES:
+            if t.mode == "state":
+                continue
+            ra, rb = _rows(ma, t, cross), _rows(mb, t, cross)
+            ma.rollback()
+            mb.rollback()
+            only_a = sorted(set(ra) - set(rb), key=str)
+            only_b = sorted(set(rb) - set(ra), key=str)
+            diff = sorted((k for k in set(ra) & set(rb) if ra[k] != rb[k]), key=str)
+            entry = {"table": t.name, "a": len(ra), "b": len(rb), "only_a": len(only_a), "only_b": len(only_b),
+                     "different": len(diff)}
+            if only_a or only_b or diff:
+                entry["samples"] = {
+                    "only_a": [ra[k] for k in only_a[:sample]],
+                    "only_b": [rb[k] for k in only_b[:sample]],
+                    "different": [(ra[k], rb[k]) for k in diff[:sample]],
+                }
+            report.append(entry)
+    finally:
+        if close_a:
+            ma.close()
+        if close_b:
+            mb.close()
+    return report
+
+
+def format_report(report: list[dict[str, Any]]) -> str:
+    lines = [f"{'table':<22}{'A':>8}{'B':>8}{'only A':>8}{'only B':>8}{'diff':>7}"]
+    for e in report:
+        flag = "" if not (e["only_a"] or e["only_b"] or e["different"]) else "   <-- differs"
+        lines.append(f"{e['table']:<22}{e['a']:>8}{e['b']:>8}{e['only_a']:>8}{e['only_b']:>8}{e['different']:>7}{flag}")
+    return "\n".join(lines)
+
+
+def identical(report: list[dict[str, Any]]) -> bool:
+    return all(not (e["only_a"] or e["only_b"] or e["different"]) for e in report)

sqlmirror/dev/e2e.py

@@ -0,0 +1,261 @@
+"""Offline end-to-end test of the whole pipeline.
+
+Runs everything locally, with no Atlassian site and no cloud account:
+
+  synthetic tenant -> mock Jira -> Forge app code (Node, mocked Forge services)
+                   -> local S3 (moto) -> sqlmirror apply -> Postgres
+  synthetic tenant -> mock Jira -> sqlmirror backfill -> Postgres
+
+and checks that the different paths produce identical databases:
+
+  1. Forge full resync + apply        == CLI backfill
+  2. after edits, deletes, new issues and links (events delivered):
+     Forge incremental + apply        == fresh backfill
+  3. backfill + Forge incremental     == fresh backfill   (the real-world setup)
+  4. after edits whose events were lost: hourly catch-up recovers them
+  5. the golden Data Center report queries all run on the mirror
+  6. reconcile --deep finds nothing to repair
+
+With a SQL Server (mssql_server_dsn), the same change files and the same mock
+Jira also load a SQL Server mirror, which must equal the Postgres one:
+
+  7. Forge files applied to SQL Server   == Postgres mirror
+  8. CLI backfill into SQL Server         == Postgres mirror
+  9. reconcile --deep on SQL Server finds nothing to repair
+ 10. the T-SQL golden reports run on SQL Server and give the same answers as
+     the Postgres golden reports on Postgres
+"""
+from __future__ import annotations
+
+import json
+import os
+import socket
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Callable
+
+import httpx
+import psycopg
+
+from ..apply import apply_once
+from ..backfill import backfill
+from ..reconcile import reconcile
+from ..db import PgMirror
+from ..jira import JiraClient
+from ..mapper import Mapper
+from ..source import S3Source
+from .compare import compare, format_report, identical
+from .golden import compare_golden, run_golden
+from .mock_jira import serve
+from .synth import build_tenant
+
+BUCKET = "sqlmirror-e2e"
+PREFIX = "site1/"
+
+
+def _free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def _dsn_for(server_dsn: str, dbname: str) -> str:
+    from psycopg.conninfo import conninfo_to_dict, make_conninfo
+
+    params = conninfo_to_dict(server_dsn)
+    params["dbname"] = dbname
+    return make_conninfo(**params)
+
+
+def _recreate_db(server_dsn: str, name: str) -> str:
+    with psycopg.connect(server_dsn, autocommit=True) as conn:
+        conn.execute(f'DROP DATABASE IF EXISTS "{name}"')
+        conn.execute(f'CREATE DATABASE "{name}"')
+    return _dsn_for(server_dsn, name)
+
+
+def run_e2e(server_dsn: str, forge_dir: str | Path, *, issues: int = 600, seed: int = 11,
+            log: Callable[[str], None] = print, mssql_server_dsn: str | None = None) -> bool:
+    import logging
+
+    logging.getLogger("werkzeug").setLevel(logging.ERROR)
+    forge_dir = Path(forge_dir).resolve()
+    if not (forge_dir / "node_modules" / "@forge" / "api").exists():
+        raise SystemExit(f"Run 'npm install' in {forge_dir} first.")
+    ok = True
+
+    def check(title: str, report) -> None:
+        nonlocal ok
+        same = identical(report)
+        ok = ok and same
+        log(("PASS " if same else "FAIL ") + title)
+        if not same:
+            log(format_report(report))
+            for e in report:
+                if e.get("samples"):
+                    log(f"  {e['table']}: {json.dumps(e['samples'], default=str)[:1500]}")
+
+    # ---------------------------------------------------------------- services
+    tenant = build_tenant(issues=issues, seed=seed)
+    jira_port, s3_port = _free_port(), _free_port()
+    server, _mock = serve(tenant, port=jira_port, background=True, flaky=0.01)
+    jira_url = f"http://127.0.0.1:{jira_port}"
+
+    os.environ.setdefault("AWS_ACCESS_KEY_ID", "testing")
+    os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "testing")
+    os.environ.setdefault("AWS_DEFAULT_REGION", "us-west-1")
+    from moto.server import ThreadedMotoServer
+
+    moto = ThreadedMotoServer(ip_address="127.0.0.1", port=s3_port, verbose=False)
+    moto.start()
+    s3_url = f"http://127.0.0.1:{s3_port}"
+    import uuid
+
+    import boto3
+
+    # moto keeps its buckets for the life of the process, so each run gets its own.
+    BUCKET = f"sqlmirror-e2e-{uuid.uuid4().hex[:8]}"
+    # Outside us-east-1, S3 needs the region spelled out when creating a bucket.
+    boto3.client("s3", endpoint_url=s3_url, region_name="us-west-1").create_bucket(
+        Bucket=BUCKET, CreateBucketConfiguration={"LocationConstraint": "us-west-1"})
+    log(f"Mock Jira on {jira_url} ({len(tenant.issues)} issues), local S3 on {s3_url}")
+
+    state_file = tempfile.NamedTemporaryFile(prefix="sqlm-forge-sim-", suffix=".json", delete=False).name
+    os.unlink(state_file)
+    env = {**os.environ, "SQLM_SIM_STATE": state_file, "SQLM_MOCK_JIRA": jira_url}
+
+    def forge(*args: str) -> str:
+        res = subprocess.run(["node", "--import", "./test/register.mjs", "test/sim.mjs", *args], cwd=forge_dir,
+                             env=env, capture_output=True, text=True, timeout=900)
+        if res.returncode != 0:
+            raise RuntimeError(f"forge sim {args[0]} failed:\n{res.stdout}\n{res.stderr}")
+        return res.stdout.strip()
+
+    def forge_state() -> dict:
+        return json.loads(forge("state"))
+
+    source = S3Source(f"s3://{BUCKET}/{PREFIX}", endpoint_url=s3_url, region="us-west-1")
+    db_forge = PgMirror(_recreate_db(server_dsn, "sqlm_e2e_forge"))
+    db_forge.init_schema()
+    db_backfill = PgMirror(_recreate_db(server_dsn, "sqlm_e2e_backfill"))
+    db_backfill.init_schema()
+    quiet = lambda _msg: None  # noqa: E731
+    mssql_dbs: list = []
+
+    try:
+        # 1. Forge full resync vs CLI backfill -------------------------------
+        forge("configure", json.dumps({"bucket": BUCKET, "region": "us-west-1", "endpoint": s3_url, "prefix": PREFIX,
+                                       "accessKeyId": "testing", "secretAccessKey": "testing"}))
+        forge("enqueue", "resync")
+        t0 = time.time()
+        log(forge("run-jobs"))
+        st = forge_state()
+        log(f"Forge resync: {st['lastJobSummary']} in {time.time() - t0:.1f}s; error: {st['lastError']}")
+        res = apply_once(source, db_forge, Mapper(), log=quiet)
+        log(f"apply: {res.files} files, {res.stats.summary()}")
+
+        client = JiraClient(jira_url, "e2e@example.com", "token")
+        backfill(client, db_backfill, Mapper(), log=quiet)
+        check("1. Forge resync + apply == CLI backfill", compare(db_forge.dsn, db_backfill.dsn))
+
+        # 2. Incremental changes with events ---------------------------------
+        events = httpx.post(f"{jira_url}/__mock/mutate", json={"edits": 40, "deletes": 4, "creates": 6, "links": 5,
+                                                              "seed": 5}).json()["events"]
+        ev_file = state_file + ".events.json"
+        Path(ev_file).write_text(json.dumps(events))
+        log(forge("events", ev_file))
+        log(forge("scheduled-sync"))
+        log(forge("run-jobs"))
+        res = apply_once(source, db_forge, Mapper(), log=quiet)
+        log(f"apply: {res.files} files, {res.stats.summary()}")
+        db_truth = PgMirror(_recreate_db(server_dsn, "sqlm_e2e_truth"))
+        db_truth.init_schema()
+        backfill(client, db_truth, Mapper(), log=quiet)
+        check("2. Forge incremental (edits, deletes, creates, links) == fresh backfill",
+              compare(db_forge.dsn, db_truth.dsn))
+
+        # 3. Backfilled mirror kept current by the Forge app -------------------
+        res = apply_once(source, db_backfill, Mapper(), log=quiet)
+        log(f"apply onto backfill: {res.files} files, {res.stats.summary()}")
+        check("3. backfill + Forge incremental == fresh backfill", compare(db_backfill.dsn, db_truth.dsn))
+
+        # 4. Lost events: the hourly catch-up must recover the edits ------------
+        httpx.post(f"{jira_url}/__mock/mutate", json={"edits": 15, "deletes": 0, "creates": 2, "links": 0, "seed": 6})
+        log(forge("scheduled-catchup"))
+        log(forge("enqueue", "sync"))  # the catch-up only queues; the next scheduled sync writes
+        log(forge("run-jobs"))
+        res = apply_once(source, db_forge, Mapper(), log=quiet)
+        db_truth2 = PgMirror(_recreate_db(server_dsn, "sqlm_e2e_truth2"))
+        db_truth2.init_schema()
+        backfill(client, db_truth2, Mapper(), log=quiet)
+        check("4. hourly catch-up recovers changes whose events were lost", compare(db_forge.dsn, db_truth2.dsn))
+
+        # 5. Golden Data Center queries ---------------------------------------
+        results = run_golden(db_forge.dsn)
+        failed = [r for r in results if not r["ok"]]
+        ok = ok and not failed
+        log(("PASS " if not failed else "FAIL ") + f"5. {len(results) - len(failed)}/{len(results)} golden DC report queries ran")
+        for r in failed:
+            log(f"   {r['name']}: {r['error']}")
+
+        # 6. Reconcile against Jira -------------------------------------------
+        def reconciled(db, title: str) -> None:
+            nonlocal ok
+            reports = reconcile(client, db, Mapper(), deep=True, log=quiet)
+            bad = [r for r in reports if not r.ok]
+            ok = ok and not bad and bool(reports)
+            log(("PASS " if reports and not bad else "FAIL ") + title)
+            for r in bad:
+                log(f"   {r.key}: jira {r.jira} mirror {r.mirror} missing {r.missing[:5]} extra {r.extra[:5]} "
+                    f"stale {r.stale[:5]}")
+
+        reconciled(db_forge, "6. reconcile --deep: Postgres mirror matches Jira")
+
+        # 7-10. SQL Server ------------------------------------------------------
+        if mssql_server_dsn:
+            from ..db_mssql import MssqlMirror, recreate_database
+
+            ms_forge = MssqlMirror(recreate_database(mssql_server_dsn, "sqlm_e2e_forge"))
+            ms_backfill = MssqlMirror(recreate_database(mssql_server_dsn, "sqlm_e2e_backfill"))
+            mssql_dbs.extend([ms_forge, ms_backfill])
+            ms_forge.init_schema()
+            ms_backfill.init_schema()
+            t0 = time.time()
+            res = apply_once(source, ms_forge, Mapper(), log=quiet)
+            log(f"SQL Server apply: {res.files} files, {res.stats.summary()} in {time.time() - t0:.1f}s")
+            again = apply_once(source, ms_forge, Mapper(), log=quiet)
+            if again.files:
+                ok = False
+                log(f"FAIL    re-running apply on SQL Server applied {again.files} files again")
+            check("7. Forge files applied to SQL Server == Postgres mirror", compare(ms_forge, db_forge))
+            t0 = time.time()
+            backfill(client, ms_backfill, Mapper(), log=quiet)
+            log(f"SQL Server backfill in {time.time() - t0:.1f}s")
+            check("8. CLI backfill into SQL Server == Postgres mirror", compare(ms_backfill, db_truth2))
+            reconciled(ms_forge, "9. reconcile --deep: SQL Server mirror matches Jira")
+
+            ms_results = run_golden(ms_forge)
+            ms_failed = [r for r in ms_results if not r["ok"]]
+            verdicts = compare_golden(results, ms_results)
+            differing = [v for v in verdicts if v["verdict"] != "same"]
+            ok = ok and not ms_failed and not differing
+            log(("PASS " if not ms_failed and not differing else "FAIL ")
+                + f"10. {len(ms_results) - len(ms_failed)}/{len(ms_results)} T-SQL golden reports ran on SQL Server; "
+                + f"{len(verdicts) - len(differing)}/{len(verdicts)} give the same answers as on Postgres")
+            for v in differing:
+                log(f"   {v['name']}: {v['verdict']}: {v['detail']}")
+
+        st = forge_state()
+        log(f"Forge totals: {json.dumps(st['totals'])}")
+    finally:
+        for db in (db_forge, db_backfill, *mssql_dbs):
+            db.close()
+        server.shutdown()
+        moto.stop()
+        for f in (state_file, state_file + ".events.json"):
+            if os.path.exists(f):
+                os.unlink(f)
+    log("ALL PASSED" if ok else "SOME CHECKS FAILED")
+    return ok