sqlmirror 0.2.0__py3-none-any.whl

sqlmirror/golden/tsql/24_cascading_select.sql

@@ -0,0 +1,13 @@
+-- Cascading select values (parentkey = '1' marks the child value)
+-- expect: rows
+-- needs: cascading select values
+SELECT cf.cfname, parent_opt.customvalue AS parent, child_opt.customvalue AS child, COUNT(*) AS issues
+FROM customfieldvalue pv
+JOIN customfield cf ON cf.id = pv.customfield
+  AND cf.customfieldtypekey = 'com.atlassian.jira.plugin.system.customfieldtypes:cascadingselect'
+JOIN customfieldoption parent_opt ON CAST(parent_opt.id AS NVARCHAR(255)) = pv.stringvalue
+LEFT JOIN customfieldvalue cv ON cv.issue = pv.issue AND cv.customfield = pv.customfield AND cv.parentkey = '1'
+LEFT JOIN customfieldoption child_opt ON CAST(child_opt.id AS NVARCHAR(255)) = cv.stringvalue
+WHERE pv.parentkey IS NULL
+GROUP BY cf.cfname, parent_opt.customvalue, child_opt.customvalue
+ORDER BY 1, 2, 3;

sqlmirror/golden/tsql/25_user_picker.sql

@@ -0,0 +1,12 @@
+-- User picker custom fields resolved to display names
+-- expect: rows
+-- needs: user picker custom field values
+SELECT cf.cfname, u.display_name, COUNT(*) AS issues
+FROM customfieldvalue cfv
+JOIN customfield cf ON cf.id = cfv.customfield
+  AND cf.customfieldtypekey IN ('com.atlassian.jira.plugin.system.customfieldtypes:userpicker',
+                                'com.atlassian.jira.plugin.system.customfieldtypes:multiuserpicker')
+JOIN app_user au ON au.user_key = cfv.stringvalue
+JOIN cwd_user u ON u.lower_user_name = au.lower_user_name
+GROUP BY cf.cfname, u.display_name
+ORDER BY issues DESC;

sqlmirror/golden/tsql/26_attachments.sql

@@ -0,0 +1,9 @@
+-- Attachment volume per project
+-- expect: rows
+-- needs: attachments
+SELECT p.pkey, COUNT(*) AS files, ROUND(SUM(fa.filesize) / 1048576.0, 1) AS megabytes
+FROM fileattachment fa
+JOIN jiraissue ji ON ji.id = fa.issueid
+JOIN project p ON p.id = ji.project
+GROUP BY p.pkey
+ORDER BY p.pkey;

sqlmirror/golden/tsql/27_project_categories.sql

@@ -0,0 +1,8 @@
+-- Projects and their categories
+-- expect: rows
+SELECT p.pkey, p.pname, pc.cname AS category
+FROM project p
+LEFT JOIN nodeassociation na ON na.source_node_id = p.id AND na.source_node_entity = 'Project'
+  AND na.association_type = 'ProjectCategory'
+LEFT JOIN projectcategory pc ON pc.id = na.sink_node_id
+ORDER BY p.pkey;

sqlmirror/golden/tsql/28_issue_types.sql

@@ -0,0 +1,7 @@
+-- Issues per issue type, sub-tasks flagged by pstyle
+-- expect: rows
+SELECT it.pname, it.pstyle, COUNT(*) AS issues
+FROM jiraissue ji
+JOIN issuetype it ON it.id = ji.issuetype
+GROUP BY it.pname, it.pstyle
+ORDER BY issues DESC;

sqlmirror/golden/tsql/29_estimate_accuracy.sql

@@ -0,0 +1,11 @@
+-- Original estimate vs time spent on resolved issues
+-- expect: rows
+-- needs: resolved issues with an estimate and logged time
+SELECT p.pkey,
+       ROUND(SUM(ji.timeoriginalestimate) / 3600.0, 1) AS estimated_hours,
+       ROUND(SUM(ji.timespent) / 3600.0, 1) AS spent_hours
+FROM jiraissue ji
+JOIN project p ON p.id = ji.project
+WHERE ji.resolutiondate IS NOT NULL AND ji.timeoriginalestimate IS NOT NULL AND ji.timespent IS NOT NULL
+GROUP BY p.pkey
+ORDER BY p.pkey;

sqlmirror/golden/tsql/30_project_leads.sql

@@ -0,0 +1,7 @@
+-- Project leads
+-- expect: rows
+SELECT p.pkey, p.pname, u.display_name AS lead
+FROM project p
+LEFT JOIN app_user au ON au.user_key = p.lead
+LEFT JOIN cwd_user u ON u.lower_user_name = au.lower_user_name
+ORDER BY p.pkey;

sqlmirror/idmap.py

@@ -0,0 +1,161 @@
+"""Map Data Center ids to Cloud ids by name.
+
+Workflow:
+  1. ``sqlmirror idmap export-sql --dialect sqlserver`` prints a query.
+  2. Run it on the Data Center database (before it is switched off) and save
+     the result as CSV with columns entity, dc_id, name, extra.
+  3. ``sqlmirror idmap build dc_export.csv`` matches each row to the mirror
+     by name and writes id_map.csv (entity, dc_id, cloud_id, name, match).
+  4. ``sqlmirror check-sql --id-map id_map.csv`` rewrites hard-coded ids.
+"""
+from __future__ import annotations
+
+import csv
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Iterable
+
+import sqlglot
+
+EXPORT_SQL = """
+SELECT 'status' AS entity, CAST(id AS VARCHAR(255)) AS dc_id, pname AS name, CAST(NULL AS VARCHAR(255)) AS extra FROM issuestatus
+UNION ALL SELECT 'priority', CAST(id AS VARCHAR(255)), pname, NULL FROM priority
+UNION ALL SELECT 'resolution', CAST(id AS VARCHAR(255)), pname, NULL FROM resolution
+UNION ALL SELECT 'issuetype', CAST(id AS VARCHAR(255)), pname, NULL FROM issuetype
+UNION ALL SELECT 'project', CAST(id AS VARCHAR(255)), pkey, pname FROM project
+UNION ALL SELECT 'customfield', CAST(id AS VARCHAR(255)), cfname, customfieldtypekey FROM customfield
+UNION ALL SELECT 'issuelinktype', CAST(id AS VARCHAR(255)), linkname, NULL FROM issuelinktype
+UNION ALL SELECT 'version', CAST(v.id AS VARCHAR(255)), v.vname, p.pkey FROM projectversion v JOIN project p ON p.id = v.project
+UNION ALL SELECT 'component', CAST(c.id AS VARCHAR(255)), c.cname, p.pkey FROM component c JOIN project p ON p.id = c.project
+UNION ALL SELECT 'option', CAST(o.id AS VARCHAR(255)), o.customvalue, cf.cfname FROM customfieldoption o JOIN customfield cf ON cf.id = o.customfield
+UNION ALL SELECT 'user', au.user_key, u.email_address, u.display_name FROM app_user au JOIN cwd_user u ON u.lower_user_name = au.lower_user_name
+UNION ALL SELECT 'sprint', CAST(s.ID AS VARCHAR(255)), s.NAME, NULL FROM AO_60DB71_SPRINT s
+""".strip()
+
+
+def export_sql(dialect: str = "postgres") -> str:
+    from .sqlcheck import DIALECTS
+
+    target = DIALECTS[dialect.lower()]
+    tree = sqlglot.parse_one(EXPORT_SQL, read="mysql")
+    if target == "postgres":
+        # Active Objects tables are quoted upper-case on Postgres.
+        from sqlglot import exp
+
+        for t in tree.find_all(exp.Table):
+            if t.name.upper().startswith("AO_"):
+                t.set("this", exp.Identifier(this=t.name.upper(), quoted=True))
+        for c in tree.find_all(exp.Column):
+            if c.table == "s":
+                c.set("this", exp.Identifier(this=c.name.upper(), quoted=True))
+    return tree.sql(dialect=target, pretty=True) + ";"
+
+
+class IdMap:
+    def __init__(self, mapping: dict[tuple[str, str], str]):
+        self.mapping = mapping
+
+    def lookup(self, entity: str, dc_id: str | None) -> str | None:
+        if dc_id is None:
+            return None
+        key = dc_id if entity == "user" else str(dc_id).strip()
+        return self.mapping.get((entity, key))
+
+    @classmethod
+    def load(cls, path: str | Path) -> "IdMap":
+        mapping = {}
+        with open(path, newline="", encoding="utf-8") as fh:
+            for row in csv.DictReader(fh):
+                if row.get("cloud_id"):
+                    mapping[(row["entity"], row["dc_id"])] = row["cloud_id"]
+        return cls(mapping)
+
+
+def _mirror_names(db: Any) -> dict[str, dict[tuple[str, ...], list[str]]]:
+    """entity -> normalized name key -> cloud ids.
+
+    ``db`` is a mirror (PgMirror / MssqlMirror) or a plain Postgres connection.
+    """
+    conn = getattr(db, "conn", db)
+    if getattr(db, "dialect", "postgres") == "tsql":
+        T = db.t  # schema-qualified, bracketed
+
+        def C(name: str) -> str:
+            return f"[{name}]"
+    else:
+        def T(name: str) -> str:
+            return f'"{name}"' if name.startswith("AO_") else name
+
+        def C(name: str) -> str:
+            return f'"{name}"' if name != name.lower() else name
+    queries = {
+        "status": f"SELECT id, pname FROM {T('issuestatus')}",
+        "priority": f"SELECT id, pname FROM {T('priority')}",
+        "resolution": f"SELECT id, pname FROM {T('resolution')}",
+        "issuetype": f"SELECT id, pname FROM {T('issuetype')}",
+        "project": f"SELECT id, pkey FROM {T('project')}",
+        "customfield": f"SELECT id, cfname FROM {T('customfield')}",
+        "issuelinktype": f"SELECT id, linkname FROM {T('issuelinktype')}",
+        "version": f"SELECT v.id, v.vname, p.pkey FROM {T('projectversion')} v JOIN {T('project')} p ON p.id = v.project",
+        "component": f"SELECT c.id, c.cname, p.pkey FROM {T('component')} c JOIN {T('project')} p ON p.id = c.project",
+        "option": f"SELECT o.id, o.customvalue, cf.cfname FROM {T('customfieldoption')} o "
+                  f"JOIN {T('customfield')} cf ON cf.id = o.customfield",
+        "sprint": f"SELECT {C('ID')}, {C('NAME')} FROM {T('AO_60DB71_SPRINT')}",
+    }
+    out: dict[str, dict[tuple[str, ...], list[str]]] = {}
+    with conn.cursor() as cur:
+        for entity, q in queries.items():
+            index: dict[tuple[str, ...], list[str]] = defaultdict(list)
+            cur.execute(q)
+            for row in cur.fetchall():
+                key = tuple((str(v) if v is not None else "").strip().lower() for v in row[1:])
+                index[key].append(str(row[0]))
+            out[entity] = index
+        users_email: dict[tuple[str, ...], list[str]] = defaultdict(list)
+        users_name: dict[tuple[str, ...], list[str]] = defaultdict(list)
+        cur.execute(f"SELECT au.user_key, u.email_address, u.display_name FROM {T('app_user')} au "
+                    f"JOIN {T('cwd_user')} u ON u.id = au.id")
+        for key, email, display in cur.fetchall():
+            if email:
+                users_email[(email.strip().lower(),)].append(key)
+            if display:
+                users_name[(display.strip().lower(),)].append(key)
+        out["user_email"] = users_email
+        out["user_name"] = users_name
+    return out
+
+
+def build(dc_rows: Iterable[dict[str, str]], db: Any) -> list[dict[str, str]]:
+    names = _mirror_names(db)
+    out = []
+    for row in dc_rows:
+        entity = (row.get("entity") or "").strip().lower()
+        dc_id = (row.get("dc_id") or "").strip()
+        name = (row.get("name") or "").strip()
+        extra = (row.get("extra") or "").strip()
+        match, cloud = "none", ""
+        if entity == "user":
+            for kind, index, key in (("email", names["user_email"], (name.lower(),)),
+                                     ("display_name", names["user_name"], (extra.lower(),))):
+                ids = index.get(key, []) if key[0] else []
+                if len(ids) == 1:
+                    match, cloud = kind, ids[0]
+                    break
+                if len(ids) > 1:
+                    match = "ambiguous"
+        elif entity in names:
+            key = (name.lower(), extra.lower()) if entity in ("version", "component", "option") else (name.lower(),)
+            ids = names[entity].get(key, [])
+            if len(ids) == 1:
+                match, cloud = "name", ids[0]
+            elif len(ids) > 1:
+                match = "ambiguous"
+        out.append({"entity": entity, "dc_id": dc_id, "cloud_id": cloud, "name": name, "match": match})
+    return out
+
+
+def write_csv(rows: list[dict[str, str]], path: str | Path) -> None:
+    with open(path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=["entity", "dc_id", "cloud_id", "name", "match"])
+        w.writeheader()
+        w.writerows(rows)

sqlmirror/jira.py

@@ -0,0 +1,223 @@
+"""Jira Cloud REST client used by the CLI (backfill, reconcile).
+
+Authenticates with the customer's own email + API token. The token is only
+ever sent to the customer's Jira site from the customer's machine.
+"""
+from __future__ import annotations
+
+import random
+import re
+import threading
+import time
+from collections import Counter
+from typing import Any, Iterator
+
+import httpx
+
+from .util import iso, utcnow
+
+ISSUE_FIELDS = ["*all"]
+RATE_HEADER_HINTS = ("ratelimit", "retry-after", "rate-limit", "beta-retry")
+_NUM = re.compile(r"/\d+")
+
+
+class JiraError(RuntimeError):
+    def __init__(self, status: int, message: str):
+        super().__init__(f"Jira returned {status}: {message}")
+        self.status = status
+
+
+class JiraClient:
+    def __init__(self, site: str, email: str | None, token: str | None, *, timeout: float = 60.0,
+                 max_retries: int = 8, verbose: bool = False):
+        site = site.rstrip("/")
+        if not site.startswith("http"):
+            site = "https://" + site
+        self.site = site
+        auth = (email, token) if email and token else None
+        self.http = httpx.Client(base_url=site, auth=auth, timeout=timeout,
+                                 headers={"Accept": "application/json", "User-Agent": "sqlmirror-cli/0.1"})
+        self.max_retries = max_retries
+        self.verbose = verbose
+        self.calls: Counter[str] = Counter()
+        self.rate_limited = 0
+        self.last_rate_headers: dict[str, str] = {}
+        self._lock = threading.Lock()
+
+    def close(self) -> None:
+        self.http.close()
+
+    # ------------------------------------------------------------- transport
+    def request(self, method: str, path: str, *, params: dict[str, Any] | None = None,
+                json: Any = None, allow_404: bool = False) -> Any:
+        attempt = 0
+        while True:
+            try:
+                resp = self.http.request(method, path, params=params, json=json)
+            except httpx.TransportError as exc:
+                attempt += 1
+                if attempt > self.max_retries:
+                    raise
+                time.sleep(min(60, 2 ** attempt) + random.random())
+                if self.verbose:
+                    print(f"  network error on {path}: {exc}; retrying")
+                continue
+            with self._lock:
+                self.calls[_NUM.sub("/{id}", path.split("?")[0])] += 1
+                hdrs = {k: v for k, v in resp.headers.items() if any(h in k.lower() for h in RATE_HEADER_HINTS)}
+                if hdrs:
+                    self.last_rate_headers = hdrs
+            if resp.status_code == 429 or resp.status_code >= 500:
+                attempt += 1
+                if resp.status_code == 429:
+                    with self._lock:
+                        self.rate_limited += 1
+                if attempt > self.max_retries:
+                    raise JiraError(resp.status_code, resp.text[:300])
+                retry_after = resp.headers.get("Retry-After")
+                try:
+                    delay = float(retry_after) if retry_after else min(60, 2 ** attempt)
+                except ValueError:
+                    delay = min(60, 2 ** attempt)
+                delay = delay * (1 + random.random() * 0.3)
+                if self.verbose:
+                    print(f"  {resp.status_code} on {path}; waiting {delay:.1f}s")
+                time.sleep(delay)
+                continue
+            if resp.status_code == 404 and allow_404:
+                return None
+            if resp.status_code >= 400:
+                raise JiraError(resp.status_code, resp.text[:500])
+            if not resp.content:
+                return None
+            return resp.json()
+
+    def get(self, path: str, **params: Any) -> Any:
+        return self.request("GET", path, params=params or None)
+
+    def post(self, path: str, body: Any) -> Any:
+        return self.request("POST", path, json=body)
+
+    # --------------------------------------------------------------- helpers
+    def paged_values(self, path: str, page_size: int = 50, **params: Any) -> Iterator[dict[str, Any]]:
+        """Iterate a PageBean endpoint (startAt / maxResults / isLast / values)."""
+        start = 0
+        while True:
+            page = self.get(path, startAt=start, maxResults=page_size, **params)
+            values = page.get("values") or []
+            yield from values
+            start += len(values)
+            if page.get("isLast", True) or not values or start >= page.get("total", 10**12):
+                return
+
+    def myself(self) -> dict[str, Any]:
+        return self.get("/rest/api/2/myself")
+
+    # ------------------------------------------------------------- metadata
+    def metadata(self) -> dict[str, Any]:
+        """Configuration objects the mirror needs. Same shape the Forge app writes."""
+        fetched = iso(utcnow())
+        projects = list(self.paged_values("/rest/api/2/project/search", 50, expand="lead,description"))
+        try:
+            priorities = list(self.paged_values("/rest/api/2/priority/search", 50))
+        except JiraError:
+            priorities = self.get("/rest/api/2/priority") or []
+        try:
+            resolutions = list(self.paged_values("/rest/api/2/resolution/search", 50))
+        except JiraError:
+            resolutions = self.get("/rest/api/2/resolution") or []
+        link_types = (self.get("/rest/api/2/issueLinkType") or {}).get("issueLinkTypes", [])
+        versions, components = {}, {}
+        for p in projects:
+            versions[p["id"]] = self.request("GET", f"/rest/api/2/project/{p['id']}/versions", allow_404=True) or []
+            components[p["id"]] = self.request("GET", f"/rest/api/2/project/{p['id']}/components",
+                                               allow_404=True) or []
+        return {
+            "v": 1, "type": "metadata", "fetchedAt": fetched,
+            "projects": projects,
+            "statuses": self.get("/rest/api/2/status") or [],
+            "priorities": priorities,
+            "resolutions": resolutions,
+            "issueTypes": self.get("/rest/api/2/issuetype") or [],
+            "fields": self.get("/rest/api/2/field") or [],
+            "issueLinkTypes": link_types,
+            "projectCategories": self.request("GET", "/rest/api/2/projectCategory", allow_404=True) or [],
+            "versions": versions,
+            "components": components,
+        }
+
+    # ---------------------------------------------------------------- issues
+    def search(self, jql: str, *, fields: list[str], expand: str | None = None,
+               page_size: int = 100) -> Iterator[list[dict[str, Any]]]:
+        """Enhanced JQL search (POST /rest/api/2/search/jql), one page at a time."""
+        token = None
+        while True:
+            body: dict[str, Any] = {"jql": jql, "fields": fields, "maxResults": page_size}
+            if expand:
+                body["expand"] = expand
+            if token:
+                body["nextPageToken"] = token
+            page = self.post("/rest/api/2/search/jql", body)
+            issues = page.get("issues") or []
+            if issues:
+                yield issues
+            token = page.get("nextPageToken")
+            if page.get("isLast", not token) or not token:
+                return
+
+    def approximate_count(self, jql: str) -> int | None:
+        try:
+            res = self.post("/rest/api/2/search/approximate-count", {"jql": jql})
+            return int(res.get("count")) if res and res.get("count") is not None else None
+        except JiraError:
+            return None
+
+    def bulkfetch(self, ids: list[str]) -> list[dict[str, Any]]:
+        if not ids:
+            return []
+        res = self.post("/rest/api/2/issue/bulkfetch",
+                        {"issueIdsOrKeys": [str(i) for i in ids], "fields": ISSUE_FIELDS, "expand": ["changelog"]})
+        return (res or {}).get("issues") or []
+
+    def complete_snapshot(self, issue: dict[str, Any], fetched_at: str) -> dict[str, Any]:
+        """Turn a search/bulkfetch result into a snapshot with complete lists."""
+        iid = str(issue["id"])
+        fields = issue.setdefault("fields", {})
+
+        changelog = issue.pop("changelog", None) or {}
+        histories = changelog.get("histories") or []
+        if changelog.get("total", len(histories)) > len(histories):
+            histories = list(self.paged_values(f"/rest/api/2/issue/{iid}/changelog", 100))
+
+        cwrap = fields.pop("comment", None) or {}
+        comments = cwrap.get("comments") or []
+        if cwrap.get("total", len(comments)) > len(comments):
+            comments = []
+            start = 0
+            while True:
+                page = self.get(f"/rest/api/2/issue/{iid}/comment", startAt=start, maxResults=100, orderBy="created")
+                batch = page.get("comments") or []
+                comments.extend(batch)
+                start += len(batch)
+                if not batch or start >= page.get("total", 0):
+                    break
+
+        wwrap = fields.pop("worklog", None) or {}
+        worklogs = wwrap.get("worklogs") or []
+        if wwrap.get("total", len(worklogs)) > len(worklogs):
+            worklogs = []
+            start = 0
+            while True:
+                page = self.get(f"/rest/api/2/issue/{iid}/worklog", startAt=start, maxResults=1000)
+                batch = page.get("worklogs") or []
+                worklogs.extend(batch)
+                start += len(batch)
+                if not batch or start >= page.get("total", 0):
+                    break
+
+        return {"v": 1, "type": "issue", "id": iid, "fetchedAt": fetched_at, "issue": issue,
+                "changelog": histories, "comments": comments, "worklogs": worklogs}
+
+    def api_call_summary(self) -> str:
+        total = sum(self.calls.values())
+        return f"{total} Jira API calls, {self.rate_limited} rate-limited responses"