sqlmirror 0.2.0__py3-none-any.whl

sqlmirror/dev/golden.py

@@ -0,0 +1,182 @@
+"""Run the golden Data Center report queries against a mirror.
+
+golden/*.sql are the reports as a Data Center customer on Postgres writes
+them; golden/tsql/*.sql are the same reports as a customer on SQL Server
+writes them (TOP, CROSS APPLY, + with CAST, DATEDIFF). Each mirror runs the
+set for its own database, and ``compare_golden`` checks that both sets give
+the same answers on the same data.
+"""
+from __future__ import annotations
+
+import math
+import time
+from datetime import date, datetime, timezone
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+GOLDEN_DIR = Path(__file__).resolve().parent.parent / "golden"
+TSQL_GOLDEN_DIR = GOLDEN_DIR / "tsql"
+
+
+def load_golden(directory: Path = GOLDEN_DIR) -> list[dict[str, Any]]:
+    out = []
+    for path in sorted(directory.glob("*.sql")):
+        text = path.read_text()
+        title = text.splitlines()[0].lstrip("- ").strip() if text.startswith("--") else path.stem
+        expect = "rows" if "-- expect: rows" in text else "any"
+        needs = None
+        for line in text.splitlines():
+            if line.startswith("-- needs:"):
+                needs = line.split(":", 1)[1].strip()
+        out.append({"name": path.stem, "title": title, "sql": text, "expect": expect, "needs": needs})
+    return out
+
+
+def _status(q: dict[str, Any], rows: list, strict: bool) -> tuple[str, str | None]:
+    if rows or q["expect"] == "any":
+        return "pass", None
+    if q["needs"] and not strict:
+        return "empty", f"no {q['needs']} on this site yet"
+    return "fail", "query returned no rows"
+
+
+def run_golden(dsn: Any, directory: Path | None = None, strict: bool = True) -> list[dict[str, Any]]:
+    """Run every golden query.
+
+    ``dsn`` is a connection string (postgresql://... or sqlserver://...) or an
+    open mirror. status: "pass" (rows returned), "empty" (no rows, and the
+    query needs data the site may simply not have; only allowed when
+    strict=False), or "fail" (SQL error, or no rows where rows are required).
+    """
+    from ..db import is_sqlserver_dsn
+
+    if not isinstance(dsn, str):
+        return _run_on_mirror(dsn, directory, strict)
+    if is_sqlserver_dsn(dsn):
+        from ..db import open_mirror
+
+        db = open_mirror(dsn)
+        try:
+            return _run_on_mirror(db, directory, strict)
+        finally:
+            db.close()
+    return _run_postgres(dsn, directory or GOLDEN_DIR, strict)
+
+
+def _run_postgres(dsn: str, directory: Path, strict: bool) -> list[dict[str, Any]]:
+    import psycopg
+
+    results = []
+    with psycopg.connect(dsn) as conn:
+        for q in load_golden(directory):
+            started = time.time()
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SET LOCAL statement_timeout = '60s'")
+                    cur.execute(q["sql"])
+                    rows = cur.fetchall()
+                conn.rollback()
+                status, error = _status(q, rows, strict)
+                results.append({**q, "ok": status != "fail", "status": status, "rows": len(rows), "sample": rows[:3],
+                                "data": rows, "error": error, "ms": int((time.time() - started) * 1000)})
+            except psycopg.Error as exc:
+                conn.rollback()
+                results.append({**q, "ok": False, "status": "fail", "rows": 0, "sample": [], "data": [],
+                                "error": str(exc).strip(), "ms": int((time.time() - started) * 1000)})
+    return results
+
+
+def _run_on_mirror(db: Any, directory: Path | None, strict: bool) -> list[dict[str, Any]]:
+    if directory is None:
+        directory = TSQL_GOLDEN_DIR if db.dialect == "tsql" else GOLDEN_DIR
+    results = []
+    for q in load_golden(directory):
+        started = time.time()
+        try:
+            rows = [tuple(r) for r in db.query(q["sql"])]
+            status, error = _status(q, rows, strict)
+            results.append({**q, "ok": status != "fail", "status": status, "rows": len(rows), "sample": rows[:3],
+                            "data": rows, "error": error, "ms": int((time.time() - started) * 1000)})
+        except Exception as exc:  # noqa: BLE001 - any driver's error
+            results.append({**q, "ok": False, "status": "fail", "rows": 0, "sample": [], "data": [],
+                            "error": str(exc).strip().splitlines()[0] if str(exc).strip() else repr(exc),
+                            "ms": int((time.time() - started) * 1000)})
+    return results
+
+
+# ------------------------------------------------------- cross-database check
+def _norm(v: Any) -> Any:
+    """One value in a form both databases agree on."""
+    if v is None:
+        return None
+    if isinstance(v, bool):
+        return int(v)
+    if isinstance(v, datetime):
+        if v.tzinfo is not None:
+            v = v.astimezone(timezone.utc).replace(tzinfo=None)
+        # SQL Server datetime keeps 1/300 s; compare at that resolution.
+        ticks = round((v - datetime(1900, 1, 1)).total_seconds() * 300)
+        return ("ts", ticks)
+    if isinstance(v, date):
+        return _norm(datetime(v.year, v.month, v.day))
+    if isinstance(v, (int, Decimal, float)):
+        f = float(v)
+        if math.isfinite(f) and f == int(f) and abs(f) < 2**53:
+            return int(f)
+        return round(f, 6)
+    return v
+
+
+def _close(a: Any, b: Any) -> bool:
+    if isinstance(a, tuple) and isinstance(b, tuple) and a and a[0] == "ts" and b and b[0] == "ts":
+        return abs(a[1] - b[1]) <= 1
+    if isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool):
+        # Reports round to one decimal; second-vs-millisecond date maths can tip the rounding.
+        return abs(a - b) <= 0.1 + 1e-9
+    return a == b
+
+
+def _rows_match(a: list[tuple], b: list[tuple]) -> bool:
+    if len(a) != len(b):
+        return False
+    key = lambda r: tuple((0, "") if x is None else (1, str(x)) for x in r)  # noqa: E731
+    left = sorted(([_norm(v) for v in r] for r in a), key=key)
+    right = sorted(([_norm(v) for v in r] for r in b), key=key)
+    if all(len(x) == len(y) and all(_close(p, q) for p, q in zip(x, y)) for x, y in zip(left, right)):
+        return True
+    # Sorting can pair rows differently when values are only "close"; fall back to greedy matching.
+    pool = list(right)
+    for row in left:
+        hit = next((i for i, cand in enumerate(pool)
+                    if len(cand) == len(row) and all(_close(p, q) for p, q in zip(row, cand))), None)
+        if hit is None:
+            return False
+        pool.pop(hit)
+    return True
+
+
+# Queries whose TOP / LIMIT cuts through ties, so which rows come back is not defined.
+TIE_BREAK_UNDEFINED = {"02_issue_keys"}
+
+
+def compare_golden(a: list[dict[str, Any]], b: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Compare two run_golden results (e.g. Postgres vs SQL Server) query by query."""
+    by_name = {r["name"]: r for r in b}
+    out = []
+    for ra in a:
+        rb = by_name.get(ra["name"])
+        if rb is None:
+            out.append({"name": ra["name"], "verdict": "missing", "detail": "no matching query"})
+            continue
+        if not (ra["ok"] and rb["ok"]):
+            out.append({"name": ra["name"], "verdict": "error", "detail": ra["error"] or rb["error"]})
+        elif ra["rows"] != rb["rows"]:
+            out.append({"name": ra["name"], "verdict": "different", "detail": f"{ra['rows']} vs {rb['rows']} rows"})
+        elif ra["name"] in TIE_BREAK_UNDEFINED or _rows_match(ra["data"], rb["data"]):
+            out.append({"name": ra["name"], "verdict": "same", "detail": f"{ra['rows']} rows"})
+        else:
+            diff = next(((x, y) for x, y in zip(sorted(ra["data"], key=str), sorted(rb["data"], key=str))
+                         if [_norm(v) for v in x] != [_norm(v) for v in y]), None)
+            out.append({"name": ra["name"], "verdict": "different", "detail": f"same row count, values differ: {diff}"})
+    return out

sqlmirror/dev/mock_jira.py

@@ -0,0 +1,347 @@
+"""A small HTTP server that imitates the Jira Cloud REST endpoints the mirror uses.
+
+It serves a synthetic tenant (see ``synth``) so the CLI backfill and the Forge
+app's sync code can run end to end without a real Jira site. It deliberately
+truncates embedded lists the way Jira does (changelog, comments, worklogs) so
+pagination code is exercised, and can inject 429 responses.
+
+Endpoints: myself, project/search, status, priority/search, resolution/search,
+issuetype, field, issueLinkType, projectCategory, project/{id}/versions,
+project/{id}/components, search/jql, search/approximate-count, issue/bulkfetch,
+issue/{id}, issue/{id}/changelog, issue/{id}/comment, issue/{id}/worklog.
+Test-only: POST /__mock/mutate, GET /__mock/stats.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import random
+import re
+import threading
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from .synth import Tenant, mutate
+
+EMBED_CHANGELOG = 40   # histories embedded in search/bulkfetch results
+EMBED_COMMENTS = 10    # comments embedded in the comment field
+EMBED_WORKLOGS = 20    # worklogs embedded in the worklog field (Jira's real limit)
+
+
+def _token(offset: int) -> str:
+    return base64.urlsafe_b64encode(str(offset).encode()).decode()
+
+
+def _offset(token: str | None) -> int:
+    if not token:
+        return 0
+    return int(base64.urlsafe_b64decode(token.encode()).decode())
+
+
+def _parse_jira_time(s: str) -> datetime:
+    return datetime.fromisoformat(s.replace("+0000", "+00:00"))
+
+
+class JqlError(ValueError):
+    pass
+
+
+_COND = re.compile(r"^\s*(\w+)\s*(=|!=|>=|<=|>|<|\bin\b|\bIN\b)\s*(.+?)\s*$", re.IGNORECASE)
+
+
+def _values(raw: str) -> list[str]:
+    raw = raw.strip()
+    if raw.startswith("("):
+        raw = raw.strip("()")
+        return [v.strip().strip('"').strip("'") for v in raw.split(",") if v.strip()]
+    return [raw.strip('"').strip("'")]
+
+
+def _relative_or_absolute(value: str, now: datetime) -> datetime:
+    m = re.fullmatch(r"-(\d+)([mhdw])", value.strip())
+    if m:
+        n, unit = int(m.group(1)), m.group(2)
+        delta = {"m": timedelta(minutes=n), "h": timedelta(hours=n), "d": timedelta(days=n), "w": timedelta(weeks=n)}
+        return now - delta[unit]
+    v = value.replace("/", "-")
+    try:
+        dt = datetime.fromisoformat(v)
+    except ValueError as exc:
+        raise JqlError(f"bad date {value}") from exc
+    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
+
+
+def compile_jql(jql: str, now: datetime | None = None):
+    """Compile the JQL subset the mirror generates into (predicate, sort key, descending)."""
+    now = now or datetime.now(timezone.utc)
+    order_field, desc = "id", False
+    m = re.search(r"\border\s+by\s+(\w+)(\s+(asc|desc))?\s*$", jql, re.IGNORECASE)
+    if m:
+        order_field = m.group(1).lower()
+        desc = (m.group(3) or "").lower() == "desc"
+        jql = jql[: m.start()]
+    preds = []
+    if jql.strip():
+        for clause in re.split(r"\s+AND\s+", jql.strip(), flags=re.IGNORECASE):
+            cm = _COND.match(clause)
+            if not cm:
+                raise JqlError(f"unsupported clause: {clause}")
+            field, op, raw = cm.group(1).lower(), cm.group(2).lower(), cm.group(3)
+            vals = _values(raw)
+            preds.append((field, op, vals))
+    if not preds:
+        raise JqlError("Unbounded JQL queries are not allowed here. Please add a search restriction to your query.")
+
+    def pred(issue: dict[str, Any]) -> bool:
+        f = issue["fields"]
+        for field, op, vals in preds:
+            if field == "project":
+                actual = [f["project"]["key"], f["project"]["id"]]
+                ok = any(v.upper() in (a.upper() for a in actual) for v in vals)
+                if op == "!=":
+                    ok = not ok
+            elif field in ("id", "issuekey", "key"):
+                if field == "id" and op in (">", ">=", "<", "<="):
+                    a, b = int(issue["id"]), int(vals[0])
+                    ok = {">": a > b, ">=": a >= b, "<": a < b, "<=": a <= b}[op]
+                else:
+                    ok = issue["id"] in vals or issue["key"] in vals
+            elif field in ("updated", "created"):
+                a = _parse_jira_time(f[field])
+                b = _relative_or_absolute(vals[0], now)
+                ok = {">": a > b, ">=": a >= b, "<": a < b, "<=": a <= b, "=": a == b}[op]
+            else:
+                raise JqlError(f"unsupported field {field}")
+            if not ok:
+                return False
+        return True
+
+    def sort_key(issue: dict[str, Any]):
+        if order_field in ("id",):
+            return int(issue["id"])
+        if order_field == "key":
+            k = issue["key"].rsplit("-", 1)
+            return (k[0], int(k[1]))
+        return issue["fields"].get(order_field) or ""
+
+    return pred, sort_key, desc
+
+
+class MockJira:
+    def __init__(self, tenant: Tenant, flaky: float = 0.0, seed: int = 1):
+        self.tenant = tenant
+        self.lock = threading.RLock()
+        self.flaky = flaky
+        self.rng = random.Random(seed)
+        self.calls: Counter[str] = Counter()
+
+    # ----------------------------------------------------------- formatting
+    def issue_json(self, iid: str, fields_req: list[str] | None, expand: list[str]) -> dict[str, Any]:
+        t = self.tenant
+        issue = t.issues[iid]
+        out: dict[str, Any] = {"expand": "renderedFields,names,schema,operations,editmeta,changelog",
+                               "id": iid, "self": f"{t.data['site']}/rest/api/2/issue/{iid}", "key": issue["key"]}
+        want_all = fields_req is None or "*all" in fields_req or "*navigable" in fields_req
+        requested = set(fields_req or [])
+        excluded = {f[1:] for f in requested if f.startswith("-")}
+        if want_all or (requested - {"id", "key"} - {f for f in requested if f.startswith("-")}):
+            full = t.full_fields(iid)
+            comments = full["comment"]["comments"]
+            if len(comments) > EMBED_COMMENTS:
+                full["comment"]["comments"] = comments[:EMBED_COMMENTS]
+                full["comment"]["maxResults"] = EMBED_COMMENTS
+            if want_all:
+                fields = {k: v for k, v in full.items() if k not in excluded}
+            else:
+                fields = {k: v for k, v in full.items() if k in requested}
+            out["fields"] = fields
+        else:
+            out["fields"] = {}
+        if "changelog" in expand:
+            hist = t.data["changelogs"].get(iid, [])
+            out["changelog"] = {"startAt": 0, "maxResults": min(len(hist), EMBED_CHANGELOG), "total": len(hist),
+                                "histories": json.loads(json.dumps(hist[:EMBED_CHANGELOG]))}
+        return out
+
+    # ------------------------------------------------------------- handlers
+    def handle(self, method: str, path: str, query: dict[str, list[str]], body: Any) -> tuple[int, Any, dict[str, str]]:
+        t = self.tenant
+        d = t.data
+        q = {k: v[0] for k, v in query.items()}
+        headers: dict[str, str] = {}
+        norm = re.sub(r"/\d+", "/{id}", path)
+        self.calls[f"{method} {norm}"] += 1
+        if self.flaky and not path.startswith("/__mock") and self.rng.random() < self.flaky:
+            return 429, {"errorMessages": ["Rate limit exceeded."]}, {"Retry-After": "1"}
+        headers.update({"X-RateLimit-Limit": "350", "X-RateLimit-Remaining": "349",
+                        "X-RateLimit-Reset": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")})
+
+        def paged(values: list[Any], default: int = 50):
+            start = int(q.get("startAt", 0))
+            size = min(int(q.get("maxResults", default)), default)
+            chunk = values[start:start + size]
+            return {"self": "", "maxResults": size, "startAt": start, "total": len(values),
+                    "isLast": start + size >= len(values), "values": chunk}
+
+        with self.lock:
+            if path == "/__mock/mutate" and method == "POST":
+                body = body or {}
+                events = mutate(t, edits=body.get("edits", 20), deletes=body.get("deletes", 3),
+                                creates=body.get("creates", 5), links=body.get("links", 4), seed=body.get("seed", 99))
+                return 200, {"events": events}, headers
+            if path == "/__mock/stats":
+                return 200, {"calls": dict(self.calls), "issues": len(t.issues)}, headers
+            if path == "/rest/api/2/mypermissions":
+                return 200, {"permissions": {"ADMINISTER": {"key": "ADMINISTER", "havePermission": True}}}, headers
+            if path == "/rest/api/2/serverInfo":
+                return 200, {"baseUrl": d["site"], "deploymentType": "Cloud", "version": "1001.0.0"}, headers
+            if path == "/rest/api/2/myself":
+                u = d["users"][0]
+                return 200, u, headers
+            if path == "/rest/api/2/project/search":
+                return 200, paged(d["projects"], 50), headers
+            if path == "/rest/api/2/status":
+                return 200, d["statuses"], headers
+            if path == "/rest/api/2/priority/search":
+                return 200, paged(d["priorities"], 50), headers
+            if path == "/rest/api/2/resolution/search":
+                return 200, paged(d["resolutions"], 50), headers
+            if path == "/rest/api/2/issuetype":
+                return 200, d["issueTypes"], headers
+            if path == "/rest/api/2/field":
+                return 200, d["fields"], headers
+            if path == "/rest/api/2/issueLinkType":
+                return 200, {"issueLinkTypes": d["issueLinkTypes"]}, headers
+            if path == "/rest/api/2/projectCategory":
+                return 200, d["projectCategories"], headers
+            m = re.fullmatch(r"/rest/api/2/project/(\d+)/(versions|components)", path)
+            if m:
+                return 200, d[m.group(2)].get(m.group(1), []), headers
+
+            if path in ("/rest/api/2/search/jql", "/rest/api/3/search/jql") and method in ("POST", "GET"):
+                body = body or {}
+                if method == "GET":
+                    body = {"jql": q.get("jql", ""), "fields": (q.get("fields") or "").split(",") if q.get("fields") else None,
+                            "expand": q.get("expand"), "maxResults": int(q.get("maxResults", 50)),
+                            "nextPageToken": q.get("nextPageToken")}
+                try:
+                    pred, key, desc = compile_jql(body.get("jql", ""))
+                except JqlError as exc:
+                    return 400, {"errorMessages": [str(exc)]}, headers
+                fields_req = body.get("fields") or ["id"]
+                if isinstance(fields_req, str):
+                    fields_req = fields_req.split(",")
+                expand = body.get("expand") or ""
+                expand = expand.split(",") if isinstance(expand, str) else list(expand)
+                matched = sorted((i for i in t.issues.values() if pred(i)), key=key, reverse=desc)
+                id_only = set(fields_req) <= {"id", "key"}
+                cap = 5000 if id_only else 100
+                size = min(int(body.get("maxResults") or 50), cap)
+                start = _offset(body.get("nextPageToken"))
+                chunk = matched[start:start + size]
+                issues = [self.issue_json(i["id"], fields_req, expand) for i in chunk]
+                is_last = start + size >= len(matched)
+                res = {"issues": issues, "isLast": is_last}
+                if not is_last:
+                    res["nextPageToken"] = _token(start + size)
+                return 200, res, headers
+
+            if path == "/rest/api/2/search/approximate-count":
+                try:
+                    pred, _, _ = compile_jql((body or {}).get("jql", ""))
+                except JqlError as exc:
+                    return 400, {"errorMessages": [str(exc)]}, headers
+                return 200, {"count": sum(1 for i in t.issues.values() if pred(i))}, headers
+
+            if path == "/rest/api/2/issue/bulkfetch" and method == "POST":
+                body = body or {}
+                wanted = [str(x) for x in body.get("issueIdsOrKeys", [])]
+                if len(wanted) > 100:
+                    return 400, {"errorMessages": ["Too many issues requested (max 100)."]}, headers
+                expand = body.get("expand") or []
+                if isinstance(expand, str):
+                    expand = expand.split(",")
+                by_key = {i["key"]: i["id"] for i in t.issues.values()}
+                issues, errors = [], []
+                for w in wanted:
+                    iid = w if w in t.issues else by_key.get(w)
+                    if iid is None:
+                        errors.append(w)
+                    else:
+                        issues.append(self.issue_json(iid, body.get("fields") or ["*navigable"], expand))
+                res = {"expand": "schema,names", "issues": issues, "issueErrors": []}
+                if errors:
+                    res["issueErrors"] = [{"issueIdsOrKeys": errors,
+                                           "errorMessages": ["Issue does not exist or you do not have permission to see it."]}]
+                return 200, res, headers
+
+            m = re.fullmatch(r"/rest/api/2/issue/(\d+)(/changelog|/comment|/worklog)?", path)
+            if m:
+                iid, sub = m.group(1), m.group(2)
+                if iid not in t.issues:
+                    return 404, {"errorMessages": ["Issue does not exist or you do not have permission to see it."]}, headers
+                if sub is None:
+                    expand = (q.get("expand") or "").split(",")
+                    fields = (q.get("fields") or "*all").split(",")
+                    return 200, self.issue_json(iid, fields, expand), headers
+                if sub == "/changelog":
+                    return 200, paged(d["changelogs"].get(iid, []), 100), headers
+                start = int(q.get("startAt", 0))
+                size = min(int(q.get("maxResults", 50)), 50)
+                if sub == "/comment":
+                    vals = d["comments"].get(iid, [])
+                    return 200, {"startAt": start, "maxResults": size, "total": len(vals),
+                                 "comments": vals[start:start + size]}, headers
+                vals = d["worklogs"].get(iid, [])
+                return 200, {"startAt": start, "maxResults": size, "total": len(vals),
+                             "worklogs": vals[start:start + size]}, headers
+        return 404, {"errorMessages": [f"mock has no route for {method} {path}"]}, headers
+
+
+def make_handler(mock: MockJira):
+    class Handler(BaseHTTPRequestHandler):
+        protocol_version = "HTTP/1.1"
+
+        def log_message(self, fmt: str, *args: Any) -> None:  # quiet
+            pass
+
+        def _do(self, method: str) -> None:
+            parsed = urlparse(self.path)
+            length = int(self.headers.get("Content-Length") or 0)
+            raw = self.rfile.read(length) if length else b""
+            body = json.loads(raw) if raw else None
+            try:
+                status, payload, headers = mock.handle(method, parsed.path, parse_qs(parsed.query), body)
+            except Exception as exc:  # surface bugs as 500s with a message
+                status, payload, headers = 500, {"errorMessages": [repr(exc)]}, {}
+            data = json.dumps(payload).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(data)))
+            for k, v in headers.items():
+                self.send_header(k, v)
+            self.end_headers()
+            self.wfile.write(data)
+
+        def do_GET(self) -> None:
+            self._do("GET")
+
+        def do_POST(self) -> None:
+            self._do("POST")
+
+    return Handler
+
+
+def serve(tenant: Tenant, host: str = "127.0.0.1", port: int = 8899, flaky: float = 0.0,
+          background: bool = False) -> tuple[ThreadingHTTPServer, MockJira]:
+    mock = MockJira(tenant, flaky=flaky)
+    server = ThreadingHTTPServer((host, port), make_handler(mock))
+    server.daemon_threads = True
+    if background:
+        threading.Thread(target=server.serve_forever, daemon=True).start()
+    else:
+        server.serve_forever()
+    return server, mock