procesio-cli 0.1.1__py3-none-any.whl

procesio_cli/tools/procesio/handlers/flowgraph.py

@@ -0,0 +1,229 @@
+"""Offline structural reader actions over a .procesio export / live flow DTO.
+
+`read-flow-graph` returns the canonical node/edge graph model (the substrate the canvas
+layout tool and inspect-flow build on). No network; reads a file, '-' (stdin), or a raw
+JSON string.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from tools.procesio.actiondef import ActionDef
+from tools.procesio import errors
+from tools.procesio.flowmodel import read_bundle, read_flow
+from tools.procesio.flowmodel.inspect import inspect as _inspect
+
+
+def _load_in(value: str) -> str:
+    """Resolve --in to a JSON string: '-' = stdin, an existing path = file, else literal."""
+    if value == "-":
+        return sys.stdin.read()
+    try:
+        p = Path(value)
+        if p.exists() and p.is_file():
+            return p.read_text(encoding="utf-8")
+    except OSError:
+        pass
+    return value
+
+
+def _in_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--in", dest="in_path", required=True,
+                   help=".procesio export or flow JSON: a path, '-' for stdin, or raw JSON")
+    p.add_argument("--flow-id", dest="flow_id",
+                   help="which flow when the bundle has more than one")
+    p.add_argument("--resource-map", dest="resource_map", action="store_true",
+                   help="emit every flow + the cross-process (process→process) edge list")
+
+
+def read_flow_graph(args) -> dict:
+    raw = _load_in(args.in_path)
+    try:
+        obj = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise errors.UsageError(f"--in is not valid JSON: {e}")
+    if args.resource_map:
+        return {"result": read_bundle(obj)}
+    return {"result": read_flow(obj, flow_id=args.flow_id).to_dict()}
+
+
+def _inspect_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--in", dest="in_path", required=True,
+                   help=".procesio export or flow JSON: a path, '-' for stdin, or raw JSON")
+    p.add_argument("--flow-id", dest="flow_id",
+                   help="which flow when the bundle has more than one")
+
+
+def inspect_flow(args) -> dict:
+    raw = _load_in(args.in_path)
+    try:
+        obj = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise errors.UsageError(f"--in is not valid JSON: {e}")
+    return {"result": _inspect(obj, flow_id=args.flow_id)}
+
+
+def _digest_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--in", dest="in_path", required=True,
+                   help="a flow JSON / API envelope / .procesio bundle file, OR a directory of them "
+                        "(every *.json is read; subprocess targets resolve to titles across the set)")
+    p.add_argument("--out", dest="out_dir",
+                   help="write one <slug>.md per flow + INDEX.md here; omit to return the Markdown inline")
+    p.add_argument("--names", dest="names",
+                   help="optional JSON file {guid: display name} for ids outside the flows "
+                        "(credentials, forms, data models)")
+
+
+def _slug(title: str) -> str:
+    import re as _re
+    s = _re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-").lower()
+    return s[:80] or "untitled"
+
+
+def flow_digest(args) -> dict:
+    from tools.procesio.flowmodel import digest as _d
+    src = Path(args.in_path)
+    paths = sorted(src.glob("*.json")) if src.is_dir() else [src]
+    if not paths:
+        raise errors.UsageError(f"--in {args.in_path}: no .json files")
+    flows = []
+    for p in paths:
+        try:
+            flows += _d.unwrap(json.loads(p.read_text(encoding="utf-8")))
+        except (json.JSONDecodeError, OSError) as e:
+            raise errors.UsageError(f"{p}: {e}")
+    names = {f.get("id"): _d._title(f) for f in flows if f.get("id")}
+    if args.names:
+        try:
+            names.update(json.loads(Path(args.names).read_text(encoding="utf-8")))
+        except (json.JSONDecodeError, OSError) as e:
+            raise errors.UsageError(f"--names: {e}")
+    if not args.out_dir:
+        return {"result": {"count": len(flows),
+                           "flows": [{"id": f.get("id"), "title": _d._title(f),
+                                      "markdown": _d.render(f, names)} for f in flows]}}
+    out = Path(args.out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    index = ["# Process index", "", "| process | nodes | calls |", "|---|---|---|"]
+    written = []
+    for f in sorted(flows, key=_d._title):
+        slug = _slug(_d._title(f)) + "-" + str(f.get("id", ""))[:8]
+        (out / f"{slug}.md").write_text(_d.render(f, names), encoding="utf-8")
+        called = [names[g] for g in _d.calls(f) if g in names and g != f.get("id")
+                  and g in {x.get("id") for x in flows}]
+        index.append(f"| [{_d._title(f)}]({slug}.md) | {len(f.get('actions') or [])} | "
+                     f"{'; '.join(sorted(set(called)))} |")
+        written.append(f"{slug}.md")
+    (out / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8")
+    return {"result": {"count": len(flows), "out": str(out), "files": written + ["INDEX.md"]}}
+
+
+def _form_digest_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--in", dest="in_path", required=True,
+                   help="a form template JSON (form-get output) or a directory of them; a sibling "
+                        "<id>.code.json (form-get-code output) adds the global CSS/JS")
+    p.add_argument("--processes", dest="processes",
+                   help="directory of process JSON (API envelope / flow) to name RUN_PROCESS targets "
+                        "and their variables")
+    p.add_argument("--out", dest="out_dir",
+                   help="write <slug>.md + <slug>/ (scripts, global form.js/form.css) per form and "
+                        "FORM-PROCESS-MAP.md here; omit to return the Markdown inline")
+
+
+def form_digest(args) -> dict:
+    from tools.procesio.flowmodel import digest as _d
+    from tools.procesio.flowmodel import formdigest as _fd
+    src = Path(args.in_path)
+    paths = ([p for p in sorted(src.glob("*.json")) if not p.name.endswith(".code.json")]
+             if src.is_dir() else [src])
+    if not paths:
+        raise errors.UsageError(f"--in {args.in_path}: no form .json files")
+    processes: dict = {}
+    if args.processes:
+        for p in sorted(Path(args.processes).glob("*.json")):
+            try:
+                for f in _d.unwrap(json.loads(p.read_text(encoding="utf-8"))):
+                    if f.get("id"):
+                        processes[f["id"]] = f
+            except (json.JSONDecodeError, OSError):
+                continue
+    forms = []
+    for p in paths:
+        try:
+            form = _fd.unwrap(json.loads(p.read_text(encoding="utf-8")))
+        except (json.JSONDecodeError, OSError) as e:
+            raise errors.UsageError(f"{p}: {e}")
+        code_path = p.with_name(p.stem + ".code.json")
+        code = {}
+        if code_path.exists():
+            try:
+                code = json.loads(code_path.read_text(encoding="utf-8"))
+            except json.JSONDecodeError:
+                code = {}
+        forms.append((form, code))
+
+    if not args.out_dir:
+        return {"result": {"count": len(forms), "forms": [
+            {"id": f.get("id"), "name": f.get("name"), "markdown": _fd.render(f, processes)[0]}
+            for f, _ in forms]}}
+
+    out = Path(args.out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    fmap = ["# Form → process map", "", "| form | element | trigger | process |", "|---|---|---|---|"]
+    written = []
+    for form, code in sorted(forms, key=lambda x: x[0].get("name") or ""):
+        slug = _slug(form.get("name") or "form") + "-" + str(form.get("id", ""))[:8]
+        md, scripts = _fd.render(form, processes)
+        side = out / slug
+        side.mkdir(exist_ok=True)
+        for i, body in enumerate(scripts, 1):
+            (side / f"script-{i:03d}.js").write_text(body, encoding="utf-8")
+        if code.get("javascript"):
+            (side / "form.js").write_text(code["javascript"], encoding="utf-8")
+        if code.get("css"):
+            (side / "form.css").write_text(code["css"], encoding="utf-8")
+        extra = []
+        if code.get("javascript") or code.get("css"):
+            extra = ["", "## Global code", "",
+                     f"- JavaScript: {len(code.get('javascript') or '')} chars → `{slug}/form.js`",
+                     f"- CSS: {len(code.get('css') or '')} chars → `{slug}/form.css`"]
+        (out / f"{slug}.md").write_text(md + "\n".join(extra) + f"\n\nScripts: `{slug}/script-NNN.js`\n",
+                                        encoding="utf-8")
+        written.append(f"{slug}.md")
+        for link in _fd.process_links(form):
+            pid = link["process_id"]
+            title = (processes.get(pid) or {}).get("title") or pid
+            fmap.append(f"| {form.get('name')} | {link['element']} | {link['trigger']} | {title} |")
+    (out / "FORM-PROCESS-MAP.md").write_text("\n".join(fmap) + "\n", encoding="utf-8")
+    return {"result": {"count": len(forms), "out": str(out), "files": written + ["FORM-PROCESS-MAP.md"]}}
+
+
+ACTIONS = {
+    "form-digest": ActionDef(
+        func=form_digest, add_args=_form_digest_args, needs_client=False,
+        description="Readable Markdown digest of forms (offline): structure tree, every element / "
+                    "form-level event, RUN_PROCESS input/output maps with form paths and process "
+                    "variables resolved to names, MAP_FORM_DATA conditions, scripts saved beside. "
+                    "--out adds FORM-PROCESS-MAP.md.",
+    ),
+    "flow-digest": ActionDef(
+        func=flow_digest, add_args=_digest_args, needs_client=False,
+        description="Readable Markdown digest of flows (offline): variable contract, nodes in "
+                    "execution order, each node's script/SQL/HTTP/subprocess parameters with "
+                    "variable and flow ids resolved to names. --in a file or a directory; --out "
+                    "writes one .md per flow + INDEX.md.",
+    ),
+    "read-flow-graph": ActionDef(
+        func=read_flow_graph, add_args=_in_args, needs_client=False,
+        description="Parse a .procesio export/flow into a node/edge graph model "
+                    "(offline). --resource-map for the cross-process graph.",
+    ),
+    "inspect-flow": ActionDef(
+        func=inspect_flow, add_args=_inspect_args, needs_client=False,
+        description="Structural summary of a flow (offline): counts, action families, "
+                    "branches, subprocess calls, resources, variables, advisory smells.",
+    ),
+}

procesio_cli/tools/procesio/handlers/flowlint.py

@@ -0,0 +1,269 @@
+"""flow-lint: replicate the designer's client-side "Process Errors" checks on a live flow.
+
+Why this exists: POST /api/Projects/validate only validates the RUNTIME layer (parameters) and
+returns EMPTY even when the designer refuses to SAVE. The designer's "Process Errors" panel is a
+separate CLIENT-SIDE check of the DESIGNER layer (customData) with NO server endpoint. This lint
+replicates that designer check so a flow can be verified offline / in CI.
+
+Checks (per action, against the DESIGNER customData layer):
+  1. SIDEPANEL_ID  — a Call/Trigger Subprocess node whose "Subprocess Configuration" side-pannel
+     setting id != the CURRENT template's id. PROCESIO bumped the template's side-pannel id over
+     time; nodes built against an older template keep the old id, so the designer can't find the
+     mapping and reports EVERY required input as "missing". THE most common cause of the phantom
+     "Mapping of required subprocess variable (X) is missing." (Networky: 1da555da / 15af8b81 vs
+     current 5456caf0 for Call, 8540605e for Trigger.)
+  2. REQ_INPUT_UNMAPPED — a required target input (type 10, isRequired) with no process-inputs row.
+  3. OUTPUT_NOT_TYPE30 — a process-outputs row mapping a target var that is NOT an output (type 30).
+     A subprocess only exposes type-30 vars as outputs; mapping a type-20 internal var => the
+     designer flags "Check data mapping."
+  4. EXECQUERY_NO_OUTPUT — an Execute Query whose Output subsection is null. The designer requires it.
+  5. CODE_ERROR_SCOPE / CODE_UNDECLARED — Node code binding an isError variable (an action's
+     error-port output, shown in the "Error" tab, never free-initialized) or an undeclared var.
+  6. missing_error_variable — variableErrorId set to a non-existent variable.
+  7. CUSTOMDATA_PLACEHOLDER — a DESIGNER setting value still holding a runtime `<%N%>` placeholder. The
+     customData layer mirrors each Parameter EXCEPT a variable ref is the raw variableId (GUID) there, not
+     the positional `<%N%>`. A `<%N%>` left in a customData value = the param->customData mirror was skipped
+     (hand-rolled flow edits bypassing the builder's _apply_values_to_config) -> the designer can't resolve
+     the token and paints it RED. (Caught three times in review before the rule was written down.)
+
+  8. UNREADABLE_STATEMENT / UNREADABLE_SCRIPT — a Query Store or Execute Query statement, or a
+     Node script, stored as ONE long line. It runs perfectly and no one can read it: a Query
+     Store node IS its statement, and the designer shows a single row that scrolls sideways off
+     the screen. The builder lays Query Store SQL out on the way in (tools/procesio/sqlformat.py);
+     this catches everything that reached the flow another way.
+
+Born 2026-07-05; rebuilt 2026-07-06 after runtime-only versions repeatedly reported false "clean".
+"""
+from __future__ import annotations
+
+import argparse
+import re
+
+from tools.procesio.actiondef import ActionDef
+from tools.procesio.client import VARIABLES_ONLY
+from tools.procesio.handlers.common import add_profile_arg
+from tools.procesio.sqlformat import is_unreadable
+
+CODE_PID = "1e6a5523-2091-6c4c-94ac-c7984074673d"
+QUERY_STORE_SQL_PID = "02577ada-0000-0200-0000-00000000b102"
+NULL_GUID = "00000000-0000-0000-0000-000000000000"
+_SUB_TEMPLATES = ("Call Subprocess", "Trigger Subprocess")
+OUTPUT_TYPE = 30  # subprocess variables exposed as outputs
+# built-in scalar datatypes share this prefix; the only object among them is ...1221 (any/Object).
+# An Execute Query Output is a SQL result SET -> must be a list<Object>: isList=True and the element
+# type an Object (the generic ...1221 OR any custom DataModel, whose GUID does NOT share this prefix).
+_PRIM_PREFIX = "0317bfee-b2f5-4bde-bfe8-1212"
+_GENERIC_OBJECT = "0317bfee-b2f5-4bde-bfe8-121212121221"
+# The DESIGNER customData mirrors each Parameter EXCEPT a variable ref is the raw variableId there,
+# while the runtime Parameter uses a <%N%> positional placeholder. A <%N%> left in a customData setting
+# value means the param->customData mirror was skipped (hand-rolled edits) -> the designer can't resolve
+# the token and paints it red ("still red <%0%>"). The builder's _apply_values_to_config does this
+# transform; this lint catches nodes that bypassed it.
+_PLACEHOLDER_RE = re.compile(r"<%\d+%>")
+
+
+def _is_guid(x) -> bool:
+    return isinstance(x, str) and len(x) == 36 and x.count("-") == 4
+
+
+def _walk_settings(action: dict):
+    """Yield every customData.configuration setting (recursing into list-valued settings)."""
+    def walk(settings):
+        for s in settings or []:
+            if isinstance(s, dict):
+                yield s
+                if isinstance(s.get("value"), list):
+                    yield from walk(s["value"])
+    for cf in (action.get("customData") or {}).get("configuration") or []:
+        yield from walk(cf.get("settings") or [])
+
+
+def _sidepanel(action: dict) -> dict | None:
+    for cf in (action.get("customData") or {}).get("configuration") or []:
+        for s in cf.get("settings") or []:
+            if s.get("type") == "side-pannel":
+                return s
+    return None
+
+
+def _subsection(sidepanel: dict | None, kind: str) -> list:
+    for sub in (sidepanel.get("value") or []) if sidepanel else []:
+        if sub.get("type") == kind:
+            return sub.get("value") or []
+    return []
+
+
+def _execquery_output(action: dict):
+    """The Execute Query Output subsection value, or the sentinel '__absent__'."""
+    for cf in (action.get("customData") or {}).get("configuration") or []:
+        for s in cf.get("settings") or []:
+            if s.get("type") == "side-pannel":
+                for sub in s.get("value") or []:
+                    if str(sub.get("label")) == "Output":
+                        return sub.get("value")
+    return "__absent__"
+
+
+_SQL_WORDS = re.compile(r"(?i)(select|insert|update|delete|merge|with)")
+
+
+def _looks_like_sql(value: str) -> bool:
+    return bool(_SQL_WORDS.search(value))
+
+
+def lint_flow_dto(flow: dict, template_sidepanel_ids: dict, target_vars_of) -> list[dict]:
+    """Pure lint over a flow DTO.
+    template_sidepanel_ids: {"Call Subprocess": id, "Trigger Subprocess": id} (current template).
+    target_vars_of(flow_id) -> {var_id: var_dict} for a called subprocess (injected: no network in tests).
+    """
+    vmap = {v["id"]: v for v in flow.get("variables") or []}
+    problems: list[dict] = []
+
+    def add(action, kind, detail):
+        problems.append({"action": action, "kind": kind, "detail": detail})
+
+    for a in flow.get("actions") or []:
+        nm = a.get("actionName")
+        tmpl = a.get("actionTemplateName")
+
+        for s in _walk_settings(a):
+            val = s.get("value")
+            if isinstance(val, str) and _PLACEHOLDER_RE.search(val):
+                add(nm, "CUSTOMDATA_PLACEHOLDER",
+                    f"designer setting '{s.get('label')}' still holds a <%N%> placeholder "
+                    f"(customData must carry the variable GUID, not the runtime placeholder - the "
+                    f"designer paints it red); param->customData mirror was not applied")
+
+        # A statement or a script stored as ONE long line runs perfectly and cannot be read:
+        # the designer shows a single row that scrolls off the screen. Flagged, not rewritten,
+        # because reflowing someone else's code is not the lint's job.
+        for prm in a.get("parameters") or []:
+            pid, val = prm.get("tabPropertyId"), prm.get("value")
+            if not isinstance(val, str) or not is_unreadable(val):
+                continue
+            if pid == QUERY_STORE_SQL_PID or (tmpl in ("Execute Query", "Execute Command")
+                                              and _looks_like_sql(val)):
+                add(nm, "UNREADABLE_STATEMENT",
+                    f"the SQL is {len(val.strip())} characters on ONE line; lay it out over "
+                    f"lines so a reader can see the clauses")
+            elif pid == CODE_PID:
+                add(nm, "UNREADABLE_SCRIPT",
+                    f"the script is {len(val.strip())} characters on ONE line; keep a script "
+                    f"body multi-line, one statement per line")
+
+        if tmpl == "Node":
+            cp = next((p for p in a.get("parameters") or [] if p.get("tabPropertyId") == CODE_PID), None)
+            for ve in (cp.get("variable") if cp else []) or []:
+                vid = ve.get("variableId")
+                v = vmap.get(vid)
+                if v is None:
+                    add(nm, "CODE_UNDECLARED", f"code binds undeclared variable {vid}")
+                elif v.get("isError"):
+                    add(nm, "CODE_ERROR_SCOPE",
+                        f"code binds error-scope variable '{v.get('name')}' (isError=true)")
+
+        if tmpl == "Execute Query":
+            out = _execquery_output(a)
+            if out is None or out == "":
+                add(nm, "EXECQUERY_NO_OUTPUT", "Execute Query Output subsection is null")
+            elif out != "__absent__":
+                ov = vmap.get(out)
+                if ov is None:
+                    add(nm, "EXECQUERY_OUTPUT_UNDECLARED", f"Output variable {out} not declared")
+                else:
+                    dt = ov.get("dataType") or ""
+                    if (not ov.get("isList")) or (dt.startswith(_PRIM_PREFIX) and dt != _GENERIC_OBJECT):
+                        add(nm, "EXECQUERY_OUTPUT_TYPE",
+                            f"Output '{ov.get('name')}' must be a list<Object> (a SQL result set) - "
+                            f"got isList={ov.get('isList')} dataType=...{dt[-4:]} "
+                            f"(scalar/primitive -> designer 'data type mismatch')")
+
+        ev = a.get("variableErrorId")
+        if ev and ev not in vmap:
+            add(nm, "missing_error_variable", f"variableErrorId {ev} not declared")
+
+        if tmpl in _SUB_TEMPLATES:
+            sp = _sidepanel(a)
+            want = template_sidepanel_ids.get(tmpl)
+            if want and (sp is None or sp.get("id") != want):
+                add(nm, "SIDEPANEL_ID",
+                    f"side-pannel id {sp.get('id') if sp else None} != current template {want} "
+                    f"(designer cannot find the mapping)")
+            target = next((p["value"] for p in a.get("parameters") or []
+                           if _is_guid(p.get("value")) and p["value"] != NULL_GUID), None)
+            if not target:
+                continue
+            tv = target_vars_of(target) or {}
+            mapped_in = {r.get("subprocess"): r.get("process") for r in _subsection(sp, "process-inputs")}
+            for vid, v in tv.items():
+                if v.get("type") == 10 and v.get("isRequired"):
+                    src = mapped_in.get(vid)
+                    if src is None or src == "":
+                        add(nm, "REQ_INPUT_UNMAPPED",
+                            f"required subprocess input '{v.get('name')}' has no designer mapping")
+                    else:
+                        root = str(src).split(".")[0]
+                        if _is_guid(root) and root not in vmap:
+                            add(nm, "REQ_INPUT_DEAD_SOURCE",
+                                f"input '{v.get('name')}' mapped from undeclared variable {root}")
+            for r in _subsection(sp, "process-outputs"):
+                ov = tv.get(r.get("subprocess"))
+                if ov is not None and ov.get("type") != OUTPUT_TYPE:
+                    add(nm, "OUTPUT_NOT_TYPE30",
+                        f"output maps target var '{ov.get('name')}' which is type {ov.get('type')}, "
+                        f"not an exposed output (type {OUTPUT_TYPE})")
+    return problems
+
+
+def _args(p: argparse.ArgumentParser) -> None:
+    add_profile_arg(p)
+    p.add_argument("--id", required=True, help="flow (process) id to lint")
+
+
+def _template_sidepanel_ids(client) -> dict:
+    acts = client.get("/api/Actions?getFullAction=true")
+    if isinstance(acts, dict):
+        acts = acts.get("result", acts)
+        acts = acts.get("actions", acts) if isinstance(acts, dict) else acts
+    out = {}
+    for tn in _SUB_TEMPLATES:
+        t = next((a for a in acts if a.get("name") == tn), None)
+        if not t:
+            continue
+        sp = next((s["id"] for tab in t.get("configuration") or []
+                   for s in tab.get("settings") or [] if s.get("type") == "side-pannel"), None)
+        if sp:
+            out[tn] = sp
+    return out
+
+
+def flow_lint(client, args) -> dict:
+    resp = client.get(f"/api/Projects/{args.id}")
+    flow = resp.get("flow", resp) if isinstance(resp, dict) else resp
+    tmpl_sp = _template_sidepanel_ids(client)
+    cache: dict = {}
+
+    def target_vars_of(fid: str) -> dict:
+        if fid not in cache:
+            try:
+                r = client.get(f"/api/Projects/{fid}", VARIABLES_ONLY)
+                f = r.get("flow", r) if isinstance(r, dict) else r
+                cache[fid] = {v["id"]: v for v in f.get("variables") or []}
+            except Exception:  # noqa: BLE001 - unreadable target -> can't assert its contract
+                cache[fid] = {}
+        return cache[fid]
+
+    problems = lint_flow_dto(flow, tmpl_sp, target_vars_of)
+    return {"id": args.id, "title": flow.get("title"), "clean": not problems,
+            "problem_count": len(problems), "problems": problems}
+
+
+ACTIONS = {
+    "flow-lint": ActionDef(
+        func=flow_lint, add_args=_args, needs_client=True,
+        description="Designer-layer 'Process Errors' lint on a live flow (the check that blocks "
+                    "designer SAVE, which POST /Projects/validate does NOT catch): stale subprocess "
+                    "side-pannel ids, unmapped/dead required subprocess inputs, subprocess outputs "
+                    "mapped to non-output (non-type-30) vars, Execute Query with null Output, and Node "
+                    "code binding error-scope/undeclared vars. clean=true when none."),
+}