sqlmirror 0.2.0__py3-none-any.whl

sqlmirror/dev/site.py

@@ -0,0 +1,383 @@
+"""Tools that write to a real (demo) Jira Cloud site, for live testing.
+
+Never point these at a production site: they create and edit issues.
+
+  load_site       create versions, components, epics, stories/tasks/bugs,
+                  sub-tasks, links, comments, worklogs and transitions
+  simulate_edits  a steady stream of realistic edits (drives Forge events)
+  bulk_edit       edit many issues quickly (the bulk-edit rate-limit test)
+"""
+from __future__ import annotations
+
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import date, timedelta
+from typing import Any, Callable
+
+from ..jira import JiraClient, JiraError
+
+WORDS = ("login checkout payment search cache api export report dashboard sync timeout crash mobile invoice "
+         "billing token session upload import permission email notification onboarding filter latency memory "
+         "index migration webhook audit").split()
+
+
+def _sentence(rng: random.Random, n: int) -> str:
+    return " ".join(rng.choice(WORDS) for _ in range(n)).capitalize()
+
+
+def ensure_project(client: JiraClient, key: str, name: str, log: Callable[[str], None]) -> dict[str, Any]:
+    existing = client.request("GET", f"/rest/api/2/project/{key}", allow_404=True)
+    if existing:
+        log(f"Using existing project {key}")
+        return existing
+    me = client.myself()
+    for template in ("com.pyxis.greenhopper.jira:gh-simplified-scrum-classic",
+                     "com.pyxis.greenhopper.jira:gh-scrum-template"):
+        try:
+            client.post("/rest/api/2/project", {"key": key, "name": name, "projectTypeKey": "software",
+                                                "projectTemplateKey": template, "leadAccountId": me["accountId"],
+                                                "assigneeType": "UNASSIGNED"})
+            log(f"Created project {key} ({template})")
+            return client.get(f"/rest/api/2/project/{key}")
+        except JiraError as exc:
+            last = exc
+    raise SystemExit(f"Could not create project {key}: {last}. Create a company-managed Scrum project with key "
+                     f"{key} in Jira, then run this again.")
+
+
+def _create_meta(client: JiraClient, key: str) -> dict[str, dict[str, Any]]:
+    """issue type name -> {"id": ..., "fields": {fieldId: field meta}}"""
+    out = {}
+    types = client.get(f"/rest/api/2/issue/createmeta/{key}/issuetypes", maxResults=50)
+    for t in types.get("issueTypes", types.get("values", [])):
+        fields = {}
+        start = 0
+        while True:
+            page = client.get(f"/rest/api/2/issue/createmeta/{key}/issuetypes/{t['id']}", startAt=start, maxResults=50)
+            vals = page.get("fields", page.get("values", []))
+            for f in vals:
+                fields[f["fieldId"]] = f
+            start += len(vals)
+            if not vals or start >= page.get("total", 0):
+                break
+        out[t["name"]] = {"id": t["id"], "subtask": t.get("subtask", False), "fields": fields}
+    return out
+
+
+def _custom_values(rng: random.Random, meta: dict[str, Any], users: list[str]) -> dict[str, Any]:
+    """Fill custom fields that are on the create screen, with values of the right shape."""
+    values: dict[str, Any] = {}
+    for fid, f in meta["fields"].items():
+        if not fid.startswith("customfield_") or rng.random() < 0.4:
+            continue
+        schema = f.get("schema") or {}
+        custom = (schema.get("custom") or "").rsplit(":", 1)[-1]
+        allowed = f.get("allowedValues") or []
+        if custom in ("select", "radiobuttons") and allowed:
+            values[fid] = {"id": rng.choice(allowed)["id"]}
+        elif custom in ("multiselect", "multicheckboxes") and allowed:
+            values[fid] = [{"id": a["id"]} for a in rng.sample(allowed, min(len(allowed), rng.randint(1, 2)))]
+        elif custom in ("float", "jsw-story-points"):
+            values[fid] = float(rng.choice([1, 2, 3, 5, 8]))
+        elif custom == "textfield":
+            values[fid] = rng.choice(["Acme Corp", "Globex", "Initech"])
+        elif custom == "textarea":
+            values[fid] = f"* {_sentence(rng, 5)}\n* {_sentence(rng, 4)}"
+        elif custom == "datepicker":
+            values[fid] = (date.today() + timedelta(days=rng.randint(5, 60))).isoformat()
+        elif custom == "labels":
+            values[fid] = [rng.choice(["vip", "regression", "q3"])]
+        elif custom == "userpicker" and users:
+            values[fid] = {"accountId": rng.choice(users)}
+    return values
+
+
+def load_site(client: JiraClient, *, project: str = "SQLM", issues: int = 500, seed: int = 1, workers: int = 4,
+              log: Callable[[str], None] = print) -> None:
+    rng = random.Random(seed)
+    proj = ensure_project(client, project, "SQL Mirror Test", log)
+    me = client.myself()
+    users = [me["accountId"]]
+
+    versions = [v["name"] for v in client.get(f"/rest/api/2/project/{project}/versions") or []]
+    for i in range(1, 4):
+        name = f"{project} {i}.0"
+        if name not in versions:
+            client.post("/rest/api/2/version", {"name": name, "projectId": int(proj["id"]), "released": i == 1,
+                                                "releaseDate": (date.today() + timedelta(days=30 * (i - 2))).isoformat()})
+    comps = [c["name"] for c in client.get(f"/rest/api/2/project/{project}/components") or []]
+    for name in ("Backend", "Frontend", "API"):
+        if name not in comps:
+            client.post("/rest/api/2/component", {"name": name, "project": project})
+    versions = client.get(f"/rest/api/2/project/{project}/versions") or []
+    components = client.get(f"/rest/api/2/project/{project}/components") or []
+
+    meta = _create_meta(client, project)
+    log(f"Issue types: {', '.join(meta)}")
+    standard = [n for n, m in meta.items() if not m["subtask"] and n.lower() != "epic"]
+    subtask_type = next((n for n, m in meta.items() if m["subtask"]), None)
+    epic_type = next((n for n in meta if n.lower() == "epic"), None)
+
+    def fields_for(type_name: str, parent_key: str | None = None) -> dict[str, Any]:
+        m = meta[type_name]
+        f: dict[str, Any] = {"project": {"key": project}, "issuetype": {"id": m["id"]},
+                             "summary": _sentence(rng, rng.randint(3, 7))}
+        if "description" in m["fields"]:
+            f["description"] = f"h2. Context\n{_sentence(rng, 10)}.\n\n* {_sentence(rng, 4)}"
+        if "labels" in m["fields"]:
+            f["labels"] = rng.sample(WORDS, rng.randint(0, 2))
+        if "priority" in m["fields"] and m["fields"]["priority"].get("allowedValues"):
+            f["priority"] = {"id": rng.choice(m["fields"]["priority"]["allowedValues"])["id"]}
+        if "components" in m["fields"] and components and rng.random() < 0.6:
+            f["components"] = [{"id": rng.choice(components)["id"]}]
+        if "fixVersions" in m["fields"] and versions and rng.random() < 0.5:
+            f["fixVersions"] = [{"id": rng.choice(versions)["id"]}]
+        if "duedate" in m["fields"] and rng.random() < 0.3:
+            f["duedate"] = (date.today() + timedelta(days=rng.randint(-20, 40))).isoformat()
+        if "timetracking" in m["fields"] and rng.random() < 0.4:
+            f["timetracking"] = {"originalEstimate": rng.choice(["1h", "2h", "4h", "1d"])}
+        if parent_key and "parent" in m["fields"]:
+            f["parent"] = {"key": parent_key}
+        f.update(_custom_values(rng, m, users))
+        return f
+
+    def create_bulk(field_sets: list[dict[str, Any]]) -> list[str]:
+        keys = []
+        for i in range(0, len(field_sets), 50):
+            res = client.post("/rest/api/2/issue/bulk", {"issueUpdates": [{"fields": f} for f in field_sets[i:i + 50]]})
+            keys.extend(x["key"] for x in res.get("issues", []))
+            for err in res.get("errors", []):
+                log(f"  create error: {str(err)[:300]}")
+        return keys
+
+    n_epics = max(1, issues // 25) if epic_type else 0
+    epic_keys = create_bulk([fields_for(epic_type) for _ in range(n_epics)]) if epic_type else []
+    log(f"Created {len(epic_keys)} epics")
+    main = []
+    for _ in range(issues - n_epics - issues // 10):
+        main.append(fields_for(rng.choice(standard), rng.choice(epic_keys) if epic_keys and rng.random() < 0.5 else None))
+    keys = create_bulk(main)
+    log(f"Created {len(keys)} issues")
+    if subtask_type and keys:
+        subs = create_bulk([fields_for(subtask_type, rng.choice(keys)) for _ in range(issues // 10)])
+        log(f"Created {len(subs)} sub-tasks")
+    else:
+        subs = []
+
+    link_types = (client.get("/rest/api/2/issueLinkType") or {}).get("issueLinkTypes", [])
+    blocks = next((t for t in link_types if t["name"] == "Blocks"), link_types[0] if link_types else None)
+
+    def enrich(key: str) -> None:
+        r = random.Random(f"{seed}-{key}")
+        for _ in range(r.choice([0, 0, 1, 2, 3])):
+            client.post(f"/rest/api/2/issue/{key}/comment", {"body": _sentence(r, r.randint(4, 12)) + "."})
+        if r.random() < 0.5:
+            transitions = client.get(f"/rest/api/2/issue/{key}/transitions").get("transitions", [])
+            for _ in range(r.choice([1, 1, 2])):
+                if not transitions:
+                    break
+                t = r.choice(transitions)
+                client.post(f"/rest/api/2/issue/{key}/transitions", {"transition": {"id": t["id"]}})
+                transitions = client.get(f"/rest/api/2/issue/{key}/transitions").get("transitions", [])
+        if r.random() < 0.3:
+            client.post(f"/rest/api/2/issue/{key}/worklog", {"timeSpentSeconds": r.choice([900, 1800, 3600]),
+                                                              "comment": _sentence(r, 3)})
+        if blocks and r.random() < 0.08 and keys:
+            other = r.choice(keys)
+            if other != key:
+                client.post("/rest/api/2/issueLink", {"type": {"name": blocks["name"]}, "inwardIssue": {"key": key},
+                                                      "outwardIssue": {"key": other}})
+
+    done = 0
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = [pool.submit(enrich, k) for k in keys + subs]
+        for f in as_completed(futures):
+            try:
+                f.result()
+            except JiraError as exc:
+                log(f"  enrich error: {exc}")
+            done += 1
+            if done % 50 == 0:
+                log(f"  enriched {done}/{len(futures)}")
+    enrich_site(client, project=project, seed=seed, workers=workers, log=log)
+
+
+def _done_and_open_transitions(client: JiraClient, key: str) -> tuple[list[dict], list[dict]]:
+    ts = client.get(f"/rest/api/2/issue/{key}/transitions").get("transitions", [])
+    done = [t for t in ts if ((t.get("to") or {}).get("statusCategory") or {}).get("key") == "done"]
+    other = [t for t in ts if t not in done]
+    return done, other
+
+
+def enrich_site(client: JiraClient, *, project: str = "SQLM", seed: int = 3, workers: int = 4,
+                log: Callable[[str], None] = print) -> None:
+    """Add the kinds of data report queries look for to existing issues:
+    assignees, Affects versions on bugs, estimates with logged time, attachments,
+    reopened issues, and sprints (closed, active and future)."""
+    rng = random.Random(seed)
+    me = client.myself()["accountId"]
+    issues = []
+    for page in client.search(f'project = "{project}" ORDER BY id ASC',
+                              fields=["issuetype", "assignee", "status", "versions"], page_size=100):
+        issues.extend(page)
+    if not issues:
+        raise SystemExit(f"No issues in {project}; run load-site first.")
+    versions = client.get(f"/rest/api/2/project/{project}/versions") or []
+    counts: dict[str, int] = {}
+    errors: dict[str, str] = {}
+
+    def note(kind: str, fn) -> None:
+        try:
+            fn()
+            counts[kind] = counts.get(kind, 0) + 1
+        except JiraError as exc:
+            errors.setdefault(kind, str(exc)[:200])
+
+    def enrich(issue: dict[str, Any]) -> None:
+        key = issue["key"]
+        f = issue.get("fields") or {}
+        r = random.Random(f"{seed}-{key}")
+        if not f.get("assignee") and r.random() < 0.7:
+            note("assigned", lambda: client.request("PUT", f"/rest/api/2/issue/{key}/assignee", json={"accountId": me}))
+        if (f.get("issuetype") or {}).get("name") == "Bug" and versions and not f.get("versions"):
+            note("affects version", lambda: client.request(
+                "PUT", f"/rest/api/2/issue/{key}", json={"fields": {"versions": [{"id": r.choice(versions)["id"]}]}}))
+        if r.random() < 0.4:
+            note("estimate", lambda: client.request(
+                "PUT", f"/rest/api/2/issue/{key}", json={"fields": {"timetracking": {"originalEstimate": r.choice(["2h", "4h", "1d"])}}}))
+            note("worklog", lambda: client.post(f"/rest/api/2/issue/{key}/worklog",
+                                                {"timeSpentSeconds": r.choice([1800, 3600, 7200]), "comment": "Work"}))
+            done, _ = _done_and_open_transitions(client, key)
+            if done:
+                note("resolved", lambda: client.post(f"/rest/api/2/issue/{key}/transitions",
+                                                     {"transition": {"id": done[0]["id"]}}))
+        if r.random() < 0.12:
+            def upload() -> None:
+                resp = client.http.post(f"/rest/api/2/issue/{key}/attachments",
+                                        files={"file": (f"{key}-notes.txt", f"Notes for {key}\n".encode(), "text/plain")},
+                                        headers={"X-Atlassian-Token": "no-check"})
+                if resp.status_code >= 400:
+                    raise JiraError(resp.status_code, resp.text[:200])
+            note("attachment", upload)
+        if r.random() < 0.08:
+            def reopen() -> None:
+                done, _ = _done_and_open_transitions(client, key)
+                if done:
+                    client.post(f"/rest/api/2/issue/{key}/transitions", {"transition": {"id": done[0]["id"]}})
+                _, back = _done_and_open_transitions(client, key)
+                if not back:
+                    raise JiraError(400, "no transition out of Done")
+                client.post(f"/rest/api/2/issue/{key}/transitions", {"transition": {"id": back[0]["id"]}})
+            note("reopened", reopen)
+
+    log(f"Enriching {len(issues)} issues in {project}...")
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        for f in as_completed([pool.submit(enrich, i) for i in issues]):
+            f.result()
+
+    # Sprints: one closed, one active, one future (needs a Scrum board).
+    try:
+        boards = client.get("/rest/agile/1.0/board", projectKeyOrId=project).get("values", [])
+        board = next((b for b in boards if b.get("type") == "scrum"), None)
+        if not board:
+            errors["sprints"] = "no Scrum board for this project (create the project from the Scrum template)"
+        else:
+            keys = [i["key"] for i in issues if not (i["fields"].get("issuetype") or {}).get("subtask")]
+            rng.shuffle(keys)
+            today = date.today()
+            plan = [("closed", -28, keys[:40]), ("active", -7, keys[40:80]), ("future", 7, keys[80:110])]
+            for n, (state, offset, chunk) in enumerate(plan, 1):
+                start = today + timedelta(days=offset)
+                sprint = client.post("/rest/agile/1.0/sprint", {
+                    "name": f"{project} Sprint {int(time.time()) % 10000}-{n}", "originBoardId": board["id"],
+                    "goal": f"Sprint {n} goal"})
+                for i in range(0, len(chunk), 50):
+                    client.post(f"/rest/agile/1.0/sprint/{sprint['id']}/issue", {"issues": chunk[i:i + 50]})
+                if state in ("active", "closed"):
+                    client.post(f"/rest/agile/1.0/sprint/{sprint['id']}", {
+                        "state": "active", "startDate": f"{start.isoformat()}T09:00:00.000Z",
+                        "endDate": f"{(start + timedelta(days=14)).isoformat()}T17:00:00.000Z"})
+                if state == "closed":
+                    client.post(f"/rest/agile/1.0/sprint/{sprint['id']}", {"state": "closed"})
+                counts[f"{state} sprint"] = len(chunk)
+    except JiraError as exc:
+        errors["sprints"] = str(exc)[:200]
+
+    log("Added: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())))
+    for kind, err in errors.items():
+        log(f"  could not add {kind}: {err}")
+    log(f"Done. {client.api_call_summary()}")
+
+
+def _issue_keys(client: JiraClient, project: str) -> list[str]:
+    keys = []
+    for page in client.search(f'project = "{project}" ORDER BY id ASC', fields=["id", "key"], page_size=5000):
+        keys.extend(i["key"] for i in page)
+    return keys
+
+
+def _edit_once(client: JiraClient, rng: random.Random, key: str) -> str:
+    kind = rng.choice(["summary", "comment", "comment", "label", "worklog", "transition"])
+    if kind == "summary":
+        client.request("PUT", f"/rest/api/2/issue/{key}", json={"fields": {"summary": _sentence(rng, 5)}})
+    elif kind == "comment":
+        client.post(f"/rest/api/2/issue/{key}/comment", {"body": _sentence(rng, 8) + "."})
+    elif kind == "label":
+        client.request("PUT", f"/rest/api/2/issue/{key}", json={"update": {"labels": [{"add": rng.choice(WORDS)}]}})
+    elif kind == "worklog":
+        client.post(f"/rest/api/2/issue/{key}/worklog", {"timeSpentSeconds": 900, "comment": "Simulated work"})
+    else:
+        transitions = client.get(f"/rest/api/2/issue/{key}/transitions").get("transitions", [])
+        if transitions:
+            client.post(f"/rest/api/2/issue/{key}/transitions", {"transition": {"id": rng.choice(transitions)["id"]}})
+    return kind
+
+
+def simulate_edits(client: JiraClient, *, project: str, per_hour: int = 300, minutes: int = 60, seed: int = 2,
+                   log: Callable[[str], None] = print) -> None:
+    """Edit random issues at a steady rate. 300/hour is a busy 250-user team."""
+    rng = random.Random(seed)
+    keys = _issue_keys(client, project)
+    if not keys:
+        raise SystemExit(f"No issues in {project}; run load-site first.")
+    interval = 3600.0 / per_hour
+    end = time.time() + minutes * 60
+    n = 0
+    log(f"Editing {per_hour} times per hour for {minutes} minutes across {len(keys)} issues. Ctrl+C to stop.")
+    while time.time() < end:
+        started = time.time()
+        try:
+            kind = _edit_once(client, rng, rng.choice(keys))
+            n += 1
+            if n % 25 == 0:
+                log(f"  {n} edits ({kind}); {client.api_call_summary()}")
+        except JiraError as exc:
+            log(f"  edit failed: {exc}")
+        time.sleep(max(0.0, interval - (time.time() - started)))
+    log(f"Done: {n} edits.")
+
+
+def bulk_edit(client: JiraClient, *, project: str, count: int = 5000, workers: int = 8,
+              log: Callable[[str], None] = print) -> None:
+    """Add one label to `count` issues as fast as the site allows (each edit fires a Jira event)."""
+    keys = _issue_keys(client, project)[:count]
+    label = f"bulk-{int(time.time())}"
+    log(f"Adding label {label} to {len(keys)} issues with {workers} workers...")
+    started = time.time()
+    done = 0
+
+    def edit(key: str) -> None:
+        client.request("PUT", f"/rest/api/2/issue/{key}", json={"update": {"labels": [{"add": label}]}})
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        for f in as_completed([pool.submit(edit, k) for k in keys]):
+            try:
+                f.result()
+            except JiraError as exc:
+                log(f"  {exc}")
+            done += 1
+            if done % 250 == 0:
+                log(f"  {done}/{len(keys)}")
+    log(f"Edited {done} issues in {time.time() - started:.0f}s. Now watch the SQL Mirror admin page: every edit "
+        "fires an event, and sync jobs fetch them 100 per Jira call.")