flowglyph 0.1.0__py3-none-any.whl

flowglyph/__init__.py

@@ -0,0 +1,70 @@
+"""FlowGlyph: enterprise automations as a vendor-neutral IR for agents.
+
+```python
+import flowglyph
+from flowglyph.views import effects_code
+
+catalog = flowglyph.extract("path/to/uipath-project")
+for capability in catalog.search("read configuration from excel", limit=5):
+    print(capability.id, effects_code(capability.effects), capability.idempotency.mode)
+catalog.save("out")
+```
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from flowglyph.adapters import Adapter, Diagnostic, Extraction, register_adapter
+from flowglyph.catalog import Catalog, Lock, extract, load_catalog
+from flowglyph.catalog.lock import Bump
+from flowglyph.catalog.versioning import Change
+from flowglyph.errors import FlowGlyphError
+from flowglyph.ir import (
+    IR_VERSION,
+    Capability,
+    Edge,
+    Effects,
+    Idempotency,
+    Node,
+    Parameter,
+    Security,
+    Semantics,
+    Source,
+    Tri,
+    Trigger,
+    Workflow,
+    export_json_schemas,
+)
+
+try:
+    __version__ = version("flowglyph")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+__all__ = [
+    "IR_VERSION",
+    "Adapter",
+    "Bump",
+    "Capability",
+    "Catalog",
+    "Change",
+    "Diagnostic",
+    "Edge",
+    "Effects",
+    "Extraction",
+    "FlowGlyphError",
+    "Idempotency",
+    "Lock",
+    "Node",
+    "Parameter",
+    "Security",
+    "Semantics",
+    "Source",
+    "Tri",
+    "Trigger",
+    "Workflow",
+    "__version__",
+    "export_json_schemas",
+    "extract",
+    "load_catalog",
+    "register_adapter",
+]

flowglyph/__main__.py

@@ -0,0 +1,5 @@
+"""Runs the command line with `python -m flowglyph`."""
+
+from flowglyph.cli import run
+
+run()

flowglyph/_internal/__init__.py

@@ -0,0 +1 @@
+"""Helpers shared across FlowGlyph. Not public API: they may change in any release."""

flowglyph/_internal/archives.py

@@ -0,0 +1,44 @@
+"""Reading zip archives from untrusted sources in memory, with limits against zip bombs."""
+
+import io
+import zipfile
+from collections.abc import Callable
+from pathlib import Path
+
+from flowglyph.errors import AdapterError
+
+MAX_ENTRY_BYTES = 20_000_000
+MAX_TOTAL_BYTES = 200_000_000
+MAX_COMPRESSION_RATIO = 200
+
+
+def read_zip(
+    source: Path | bytes, name: str, wanted: Callable[[str], bool] = lambda _: True
+) -> dict[str, bytes]:
+    """The entries of a zip archive that `wanted` accepts, read without touching the disk.
+
+    Raises:
+        AdapterError: If the archive is corrupt, or an entry is too large or compresses too well
+            to be read safely.
+    """
+    try:
+        archive = zipfile.ZipFile(io.BytesIO(source) if isinstance(source, bytes) else source)
+    except (zipfile.BadZipFile, OSError) as error:
+        raise AdapterError(f"{name}: {error}") from error
+    files: dict[str, bytes] = {}
+    total = 0
+    with archive:
+        for info in archive.infolist():
+            if info.is_dir() or not wanted(info.filename):
+                continue
+            ratio = info.file_size / max(info.compress_size, 1)
+            total += info.file_size
+            if info.file_size > MAX_ENTRY_BYTES or ratio > MAX_COMPRESSION_RATIO:
+                raise AdapterError(f"{name}: {info.filename} is too large to read safely")
+            if total > MAX_TOTAL_BYTES:
+                raise AdapterError(f"{name}: the archive is too large to read safely")
+            try:
+                files[info.filename] = archive.read(info)
+            except (zipfile.BadZipFile, OSError, RuntimeError) as error:
+                raise AdapterError(f"{name}: {info.filename}: {error}") from error
+    return files

flowglyph/_internal/graph.py

@@ -0,0 +1,45 @@
+"""Graph steps every adapter needs: following flows through folded nodes, and marking joins."""
+
+from collections import Counter
+from collections.abc import Collection, Iterable, Mapping, Sequence
+
+from flowglyph.ir import Edge, Node
+
+type Flow = tuple[str, str | None]
+"""Where a flow goes and under which condition, if any."""
+
+
+def follow(outgoing: Mapping[str, Sequence[Flow]], kept: Collection[str]) -> list[Edge]:
+    """Edges between kept nodes, following flows through the nodes that were folded away.
+
+    A flow keeps the first condition met on its way, so two branches that meet again in a
+    folded node still reach the next kept node as two edges, each with its own condition.
+    """
+    edges: set[Edge] = set()
+    for start in kept:
+        seen: set[Flow] = set()
+        pending = list(outgoing.get(start, ()))
+        while pending:
+            target, condition = pending.pop()
+            if target in kept:
+                edges.add(Edge(source=start, target=target, condition=condition))
+            elif (target, condition) not in seen:
+                seen.add((target, condition))
+                pending.extend(
+                    (next_target, condition or next_condition)
+                    for next_target, next_condition in outgoing.get(target, ())
+                )
+    return sorted(edges, key=lambda edge: (edge.source, edge.target, edge.condition or ""))
+
+
+def with_joins(
+    nodes: Iterable[Node], edges: Iterable[Edge], parallel: Collection[str] = ()
+) -> list[Node]:
+    """Nodes where several edges meet wait for any of them, unless they join parallel branches."""
+    incoming = Counter(edge.target for edge in edges)
+    return [
+        node.model_copy(update={"join": "any"})
+        if incoming[node.id] > 1 and node.id not in parallel
+        else node
+        for node in nodes
+    ]

flowglyph/_internal/hashing.py

@@ -0,0 +1,16 @@
+"""Content hashes for lock files, stable across runs and machines."""
+
+import hashlib
+import json
+from typing import Any
+
+
+def digest(data: bytes) -> str:
+    """Short, stable content hash used in lock files."""
+    return "sha256:" + hashlib.sha256(data).hexdigest()[:32]
+
+
+def fingerprint(value: Any) -> str:
+    """Hash of a JSON-serialisable value, independent of key order."""
+    canonical = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return digest(canonical.encode())

flowglyph/_internal/markdown.py

@@ -0,0 +1,209 @@
+"""The L1 representation: Markdown cards for capabilities and the catalog index."""
+
+import json
+import re
+from collections import defaultdict
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+from flowglyph._internal.text import first_line
+from flowglyph.ir import Capability, Parameter, Tri, Workflow
+from flowglyph.views import effects_code, retry_advice
+
+if TYPE_CHECKING:
+    from flowglyph.catalog import Catalog
+
+_UNSAFE_FILENAME = re.compile(r"[^\w.-]+")
+_EFFECTS_LEGEND = (
+    "Effects codes: R reads, W mutates state, D destructive, X external. "
+    "Upper case means yes, lower case means unknown."
+)
+
+
+def card_filename(capability: Capability) -> str:
+    """File name of the Markdown card of `capability`."""
+    return _UNSAFE_FILENAME.sub("_", capability.id).strip("_") + ".md"
+
+
+def _cell(value: object) -> str:
+    return str(value).replace("|", "\\|").replace("\n", " ")
+
+
+def _effects_line(capability: Capability) -> str:
+    effects = capability.effects
+    parts = [
+        f"{name}: {value.value}"
+        for name, value in (
+            ("reads", effects.reads),
+            ("mutates state", effects.mutates_state),
+            ("destructive", effects.destructive),
+            ("external", effects.external),
+        )
+    ]
+    return ", ".join(parts)
+
+
+def _trigger_line(capability: Capability) -> str:
+    trigger = capability.trigger
+    if trigger is None:
+        return ""
+    kind = {"call": "a call", "schedule": "a schedule", "event": "an event"}[trigger.kind]
+    via = f" through {trigger.channel}" if trigger.channel else ""
+    at = f" at `{trigger.address}`" if trigger.address else ""
+    return f"{kind}{via}{at}"
+
+
+def _idempotency_line(capability: Capability) -> str:
+    idempotency = capability.idempotency
+    return (
+        f"{idempotency.mode}, {retry_advice(capability.idempotency)} "
+        f"({idempotency.provenance}, confidence {idempotency.confidence:.1f})"
+    )
+
+
+def _parameter_table(parameters: list[Parameter], *, inputs: bool) -> list[str]:
+    if not parameters:
+        return ["None."]
+    header = (
+        "| Name | Type | Required | Description |" if inputs else "| Name | Type | Description |"
+    )
+    rows = [header, "|---|---|---|---|" if inputs else "|---|---|---|"]
+    for parameter in parameters:
+        description = _cell(parameter.description or "")
+        if inputs:
+            required = "yes" if parameter.required else "no"
+            rows.append(f"| `{parameter.name}` | {_type(parameter)} | {required} | {description} |")
+        else:
+            rows.append(f"| `{parameter.name}` | {_type(parameter)} | {description} |")
+    return rows
+
+
+def _graph_section(workflow: Workflow, capabilities: Mapping[str, Capability]) -> list[str]:
+    lines = ["## Steps", ""]
+    for position, node in enumerate(workflow.nodes, start=1):
+        target = capabilities.get(node.capability) if node.capability else None
+        reference = f" -> [{target.title}]({card_filename(target)})" if target else ""
+        join = ", runs if any predecessor ran" if node.join == "any" else ""
+        lines.append(f"{position}. **{_cell(node.label)}** ({node.kind}{join}){reference}")
+        if node.note:
+            lines.append(f"   {_cell(first_line(node.note, 240))}")
+    conditional = [edge for edge in workflow.edges if edge.condition]
+    if conditional:
+        lines += ["", "Branches:", ""]
+        lines += [f"- `{e.source}` -> `{e.target}` when `{e.condition}`" for e in conditional]
+    if workflow.unresolved_calls:
+        lines += ["", "Unresolved invocations:", ""]
+        lines += [f"- `{call}`" for call in workflow.unresolved_calls]
+    return [*lines, ""]
+
+
+def render_capability(
+    capability: Capability,
+    workflows: Mapping[str, Workflow],
+    capabilities: Mapping[str, Capability],
+) -> str:
+    """The L1 card of a capability: what an agent reads before filling its arguments."""
+    front_matter = {
+        "id": capability.id,
+        "kind": capability.kind,
+        "category": capability.category,
+        "version": capability.version,
+        "effects": effects_code(capability.effects) or "none",
+        "idempotency": capability.idempotency.mode,
+        "invocable": capability.invocable,
+    }
+    lines = ["---", *(f"{key}: {json.dumps(value)}" for key, value in front_matter.items()), "---"]
+    lines += ["", f"# {capability.title}", ""]
+    if capability.description:
+        lines += [capability.description, ""]
+    if capability.generated:
+        written = " and ".join(capability.generated)
+        lines += [f"_The {written} were written by a language model: the source did not say._", ""]
+    lines += [
+        f"- **Source:** `{capability.source.location}` ({capability.source.provider})",
+        f"- **Effects:** {_effects_line(capability)}",
+        f"- **Idempotency:** {_idempotency_line(capability)}",
+    ]
+    if capability.trigger is not None:
+        lines.append(f"- **Started by:** {_trigger_line(capability)}")
+    if capability.configuration:
+        names = ", ".join(f"`{name}`" for name in capability.configuration)
+        lines.append(f"- **Configuration:** {names}")
+    if capability.deprecated:
+        lines.append("- **Deprecated** in its source")
+    if capability.security.approval_required:
+        lines.append("- **Approval:** required before every run")
+    lines.append("")
+    if capability.notes:
+        lines += ["> **Notes**", *(f"> - {note}" for note in capability.notes), ""]
+    lines += ["## Inputs", "", *_parameter_table(capability.inputs, inputs=True), ""]
+    if capability.kind != "activity":
+        lines += ["## Outputs", "", *_parameter_table(capability.outputs, inputs=False), ""]
+    workflow = workflows.get(capability.id)
+    if workflow is not None:
+        lines += _graph_section(workflow, capabilities)
+    if capability.used_in:
+        lines += ["## Used in", "", *(f"- `{location}`" for location in capability.used_in), ""]
+    lines += [
+        "## Contract",
+        "",
+        "```json",
+        json.dumps(
+            {"input": capability.input_schema, "output": capability.output_schema},
+            indent=2,
+            ensure_ascii=False,
+        ),
+        "```",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def render_index(catalog: "Catalog") -> str:
+    """The catalog's front page: every workflow and activity at a glance."""
+    workflows = sorted(
+        (c for c in catalog if c.kind != "activity"), key=lambda capability: capability.id
+    )
+    activities = sorted((c for c in catalog if c.kind == "activity"), key=lambda c: c.title)
+    lines = [
+        "# Catalog",
+        "",
+        f"{len(workflows)} invocable capabilities, {len(activities)} activity types.",
+        "",
+        "## Invocable",
+        "",
+        "| Capability | Description | Inputs | Effects | Idempotency | Version |",
+        "|---|---|---|---|---|---|",
+    ]
+    for capability in workflows:
+        inputs = ", ".join(f"`{parameter.name}`" for parameter in capability.inputs) or "none"
+        lines.append(
+            f"| [{capability.title}](capabilities/{card_filename(capability)}) "
+            f"| {_cell(first_line(capability.description, 100))} | {inputs} "
+            f"| {effects_code(capability.effects) or 'none'} | {capability.idempotency.mode} "
+            f"| {capability.version} |"
+        )
+    by_category: defaultdict[str, list[Capability]] = defaultdict(list)
+    for capability in activities:
+        by_category[capability.category].append(capability)
+    if by_category:
+        lines += ["", "## Activity types", ""]
+    for category in sorted(by_category):
+        lines += [f"### {category}", ""]
+        for capability in by_category[category]:
+            flag = " (side effects)" if capability.effects.has_side_effects else ""
+            mutates = capability.effects.mutates_state
+            certainty = " (unknown)" if mutates is Tri.UNKNOWN else ""
+            lines.append(
+                f"- [{capability.title}](capabilities/{card_filename(capability)}){flag}{certainty}"
+            )
+        lines.append("")
+    lines += [
+        _EFFECTS_LEGEND,
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def _type(parameter: Parameter) -> str:
+    return f"{parameter.type} or null" if parameter.nullable else parameter.type

flowglyph/_internal/paths.py

@@ -0,0 +1,20 @@
+"""Short names that tell apart artifacts sharing a name in different folders."""
+
+import re
+from pathlib import PurePosixPath
+
+_UNSAFE = re.compile(r"[^A-Za-z0-9.-]+")
+
+
+def distinguishing_names(paths: list[str]) -> list[str]:
+    """The shortest path segment that tells each of several files apart.
+
+    `["v2/API/API.edmx", "v4/API/API.edmx"]` gives `["v2", "v4"]`; files in one folder fall back
+    to their names.
+    """
+    parts = [PurePosixPath(path).with_suffix("").parts for path in paths]
+    for index in range(max(len(p) for p in parts)):
+        segments = [p[index] if index < len(p) else "" for p in parts]
+        if len(set(segments)) == len(segments):
+            return [_UNSAFE.sub("_", segment).strip("_") for segment in segments]
+    return [_UNSAFE.sub("_", "/".join(p)).strip("_") for p in parts]

flowglyph/_internal/search.py

@@ -0,0 +1,67 @@
+"""BM25 ranking over capabilities, with weighted fields and no dependencies."""
+
+import math
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from flowglyph._internal.text import tokenize
+
+
+@dataclass(frozen=True)
+class Document:
+    """A searchable text with its fields weighted by repetition."""
+
+    key: str
+    fields: Sequence[tuple[str, int]]
+
+
+@dataclass
+class BM25Index:
+    """Okapi BM25 over a fixed set of documents.
+
+    Small enough to avoid a dependency and fast enough for catalogs of tens of thousands of
+    capabilities.
+    """
+
+    documents: Sequence[Document]
+    k1: float = 1.5
+    b: float = 0.75
+    _frequencies: list[Counter[str]] = field(init=False, default_factory=list[Counter[str]])
+    _idf: dict[str, float] = field(init=False, default_factory=dict[str, float])
+    _average_length: float = field(init=False, default=0.0)
+
+    def __post_init__(self) -> None:
+        for document in self.documents:
+            counts: Counter[str] = Counter()
+            for text, weight in document.fields:
+                for token in tokenize(text):
+                    counts[token] += weight
+            self._frequencies.append(counts)
+        total = len(self._frequencies) or 1
+        self._average_length = sum(sum(c.values()) for c in self._frequencies) / total
+        document_frequency: Counter[str] = Counter()
+        for counts in self._frequencies:
+            document_frequency.update(counts.keys())
+        self._idf = {
+            token: math.log(1 + (total - seen + 0.5) / (seen + 0.5))
+            for token, seen in document_frequency.items()
+        }
+
+    def search(self, query: str, limit: int = 10) -> list[tuple[str, float]]:
+        """Keys of the best matching documents with their scores, best first."""
+        terms = tokenize(query)
+        scored: list[tuple[str, float]] = []
+        for document, counts in zip(self.documents, self._frequencies, strict=True):
+            length = sum(counts.values())
+            score = 0.0
+            for term in terms:
+                frequency = counts.get(term, 0)
+                if not frequency:
+                    continue
+                normaliser = self.k1 * (1 - self.b + self.b * length / (self._average_length or 1))
+                score += self._idf[term] * frequency * (self.k1 + 1) / (frequency + normaliser)
+            if score > 0:
+                scored.append((document.key, score))
+        scored.sort(key=lambda item: (-item[1], item[0]))
+        return scored[:limit]

flowglyph/_internal/text.py

@@ -0,0 +1,42 @@
+"""Tokenizing and trimming text for search and summaries."""
+
+import re
+
+_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
+_WORD = re.compile(r"[^\W_]+")
+_ARGUMENT_PREFIX = re.compile(r"^(in|out|io)_")
+_SHORTEST_PLURAL = 4
+
+
+def tokenize(text: str) -> list[str]:
+    """Lower-case search tokens, splitting camelCase and snake_case identifiers.
+
+    UiPath argument prefixes (`in_`, `out_`, `io_`) are dropped so that `in_ConfigFile`
+    matches a search for "config file".
+    """
+    tokens: list[str] = []
+    for raw in text.split():
+        for word in _WORD.findall(_ARGUMENT_PREFIX.sub("", raw)):
+            tokens.extend(
+                _singular(part.lower()) for part in _CAMEL_BOUNDARY.split(word) if len(part) > 1
+            )
+    return tokens
+
+
+def _singular(token: str) -> str:
+    """Fold simple English plurals so "duplicates" matches "Duplicate"."""
+    if len(token) > _SHORTEST_PLURAL and token.endswith("ies"):
+        return token[:-3] + "y"
+    if (
+        len(token) >= _SHORTEST_PLURAL
+        and token.endswith("s")
+        and not token.endswith(("ss", "us", "is"))
+    ):
+        return token[:-1]
+    return token
+
+
+def first_line(text: str, limit: int = 120) -> str:
+    """The first line of `text`, cut to `limit` characters."""
+    line = text.strip().split("\n", 1)[0].strip()
+    return line if len(line) <= limit else line[: limit - 1].rstrip() + "…"

flowglyph/_internal/xml.py

@@ -0,0 +1,47 @@
+"""XML parsing for untrusted sources: documents that declare a DTD are refused."""
+
+from pathlib import Path
+from xml.etree import ElementTree as ET
+from xml.parsers import expat
+
+from flowglyph.errors import AdapterError
+
+_LAST_CHUNK = True
+
+
+def load_xml(path: Path) -> ET.Element:
+    """Parse an XML file, refusing documents that declare a DTD.
+
+    Raises:
+        AdapterError: If the file is not well-formed XML or declares a DTD.
+    """
+    return parse_xml(path.read_bytes(), str(path))
+
+
+def parse_xml(data: bytes, name: str) -> ET.Element:
+    """Parse XML bytes, refusing documents that declare a DTD.
+
+    Raises:
+        AdapterError: If the data is not well-formed XML or declares a DTD.
+    """
+    _refuse_dtd(data, name)
+    try:
+        # ElementTree never resolves external entities, and with DTDs refused there are no
+        # internal entities to expand, so this input is safe to parse.
+        return ET.fromstring(data)  # noqa: S314
+    except ET.ParseError as error:
+        raise AdapterError(f"{name}: {error}") from error
+
+
+def _refuse_dtd(data: bytes, name: str) -> None:
+    """Scan the whole document with expat, which reports a DOCTYPE wherever it appears."""
+
+    def refuse(*_: object) -> None:
+        raise AdapterError(f"{name}: XML documents must not declare a DTD")
+
+    scanner = expat.ParserCreate()
+    scanner.StartDoctypeDeclHandler = refuse
+    try:
+        scanner.Parse(data, _LAST_CHUNK)
+    except expat.ExpatError as error:
+        raise AdapterError(f"{name}: {error}") from error