libre-devops-helpers 0.4.1__py3-none-any.whl

libre_devops_helpers/microsoft/loganalytics/client.py

@@ -0,0 +1,117 @@
+"""Run KQL against a Log Analytics (or Sentinel) workspace through the query API.
+
+Access rests on Azure RBAC on the workspace (Log Analytics Reader is enough), not on
+token scopes. The workspace is named by its workspace id (the GUID the portal calls the
+"Workspace ID"), not by its resource id.
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Self
+
+import requests
+
+from libre_devops_helpers.core.auth import TokenProvider, token_source
+from libre_devops_helpers.core.errors import ApiError, LdoError
+from libre_devops_helpers.core.http import ApiClient
+from libre_devops_helpers.core.tables import QueryResult
+from libre_devops_helpers.core.util import is_guid
+from libre_devops_helpers.microsoft.clouds import PUBLIC
+from libre_devops_helpers.microsoft.config import Profile
+
+
+class LogAnalyticsClient:
+    """Log Analytics queries. Close it (or use ``with``) when done."""
+
+    def __init__(self, api: ApiClient) -> None:
+        self.api = api
+
+    @classmethod
+    def create(
+        cls,
+        tokens: TokenProvider,
+        tenant_id: str,
+        *,
+        api_url: str = PUBLIC.log_analytics_url,
+        verify: bool | str = True,
+        session: requests.Session | None = None,
+    ) -> LogAnalyticsClient:
+        """A client for ``tenant_id`` that takes its tokens from ``tokens``."""
+        api = ApiClient(
+            api_url,
+            token_source(tokens, api_url, tenant_id),
+            name="Log Analytics",
+            verify=verify,
+            session=session,
+            # A query can legitimately run for minutes; the server caps it at 10.
+            timeout=600.0,
+        )
+        return cls(api)
+
+    @classmethod
+    def for_profile(
+        cls,
+        profile: Profile,
+        tokens: TokenProvider,
+        *,
+        verify: bool | str = True,
+        session: requests.Session | None = None,
+    ) -> LogAnalyticsClient:
+        """A client for a configured profile's tenant, in the profile's cloud."""
+        return cls.create(
+            tokens,
+            profile.tenant_id,
+            api_url=profile.cloud.log_analytics_url,
+            verify=verify,
+            session=session,
+        )
+
+    def close(self) -> None:
+        self.api.close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    def query(
+        self, workspace_id: str, query: str, *, timespan: timedelta | None = None
+    ) -> QueryResult:
+        """Run ``query`` and return its first table.
+
+        ``timespan`` bounds the query from now backwards, on top of any time filter in
+        the query itself. A partial result comes back with the service's error in
+        ``warnings`` rather than failing.
+        """
+        if not is_guid(workspace_id):
+            raise LdoError(
+                f"not a Log Analytics workspace id: {workspace_id!r}",
+                hint="use the workspace's Workspace ID (a GUID), not its resource id",
+            )
+        if not query.strip():
+            raise LdoError("the Log Analytics query is empty")
+        body: dict[str, Any] = {"query": query}
+        if timespan is not None:
+            body["timespan"] = f"PT{int(timespan.total_seconds())}S"
+        data = self.api.post(f"/v1/workspaces/{workspace_id.strip().lower()}/query", body)
+        tables = data.get("tables")
+        warnings: list[str] = []
+        error = data.get("error")
+        if isinstance(error, dict):
+            detail = str(error.get("message") or error.get("code") or "partial result")
+            if not isinstance(tables, list) or not tables:
+                raise ApiError(f"Log Analytics: {detail}", code=str(error.get("code") or ""))
+            warnings.append(detail)
+        if not isinstance(tables, list) or not tables or not isinstance(tables[0], dict):
+            return QueryResult((), (), warnings=tuple(warnings))
+        table = tables[0]
+        columns = [
+            str(column.get("name"))
+            for column in table.get("columns") or []
+            if isinstance(column, dict)
+        ]
+        rows = [row for row in table.get("rows") or [] if isinstance(row, list)]
+        result = QueryResult.from_columns(columns, rows)
+        return QueryResult(result.columns, result.rows, warnings=tuple(warnings))

libre_devops_helpers/microsoft/logicapps/__init__.py

@@ -0,0 +1,79 @@
+"""Consumption Logic App workflows: offline checks, Azure's own validation, export.
+
+Ported from the LibreDevOpsHelpers LogicApps module. The offline half checks a
+definition against the contract Azure enforces, with no network; the online half reads
+deployed workflows and asks the resource provider to validate a definition without
+deploying it.
+
+Public API::
+
+    from libre_devops_helpers.microsoft.logicapps import check, load
+
+    findings = check(load(Path("router.json")), connections=["office365"])
+"""
+
+from libre_devops_helpers.microsoft.logicapps.checks import (
+    Connection,
+    ConnectionReference,
+    DeployStep,
+    Difference,
+    Finding,
+    ParameterStatus,
+    check,
+    compare,
+    connection_references,
+    connections_of,
+    deploy_order,
+    parameter_status,
+    rewrite_references,
+    with_parameter_defaults,
+)
+from libre_devops_helpers.microsoft.logicapps.client import (
+    API_VERSION,
+    LogicAppsClient,
+    Validation,
+)
+from libre_devops_helpers.microsoft.logicapps.document import (
+    SHAPES,
+    TOKEN_MARK,
+    ActionNode,
+    WorkflowDocument,
+    action_nodes,
+    load,
+    parse,
+    token_safe_json,
+    workflow_name_from,
+)
+
+# ARM access rests on Azure RBAC (Logic App Reader, or Reader), not token scopes.
+REQUIREMENTS: tuple = ()
+
+__all__ = [
+    "API_VERSION",
+    "REQUIREMENTS",
+    "SHAPES",
+    "TOKEN_MARK",
+    "ActionNode",
+    "Connection",
+    "ConnectionReference",
+    "DeployStep",
+    "Difference",
+    "Finding",
+    "LogicAppsClient",
+    "ParameterStatus",
+    "Validation",
+    "WorkflowDocument",
+    "action_nodes",
+    "check",
+    "compare",
+    "connection_references",
+    "connections_of",
+    "deploy_order",
+    "load",
+    "parameter_status",
+    "parse",
+    "rewrite_references",
+    "token_safe_json",
+    "with_parameter_defaults",
+    "workflow_name_from",
+]

libre_devops_helpers/microsoft/logicapps/checks.py

@@ -0,0 +1,432 @@
+"""Offline checks on workflow definitions, against the contract Azure enforces.
+
+These are not matters of taste; each is something the resource provider rejects, or
+that deploys cleanly and fails when the workflow runs:
+
+- Every declared parameter needs a value by deploy time, or the engine returns
+  ``InvalidTemplate`` ("The value for the workflow parameter ... is not provided").
+  ``$connections`` is the exception a deployment tool generates. SecureString and
+  SecureObject should arrive as inputs, so a secret never lands in the definition.
+- A connection is referenced by an arbitrary key, and that key has to match the one
+  wired at deploy time; nothing checks it until a run fails.
+- A native Workflow dispatch action's target is checked when the caller is written
+  (``NestedWorkflowNotFound``), so a caller deploys after the workflow it calls.
+
+Nothing here touches the network.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import re
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from libre_devops_helpers.core.errors import InputError
+from libre_devops_helpers.microsoft.logicapps.document import WorkflowDocument, action_nodes
+
+SECURE_TYPES = frozenset({"SecureString", "SecureObject"})
+CONNECTIONS = "$connections"
+_REFERENCE = re.compile(r"\$connections'\)\[\s*'([^']+)'\s*\]")
+
+
+@dataclass(frozen=True)
+class ParameterStatus:
+    """Whether one declared parameter will have a value at deploy time, and from where."""
+
+    workflow: str | None
+    name: str
+    type: str | None
+    secure: bool
+    satisfied: bool
+    satisfied_by: str  # generated, supplied, default, wrapper, or none
+    reason: str
+    source: str
+
+
+@dataclass(frozen=True)
+class Finding:
+    """Something wrong with a definition: an error Azure rejects, or a warning."""
+
+    workflow: str | None
+    severity: str  # error or warning
+    rule: str
+    message: str
+    source: str
+
+
+@dataclass(frozen=True)
+class Connection:
+    """A managed API connection a wrapper resolves, and how it authenticates."""
+
+    workflow: str | None
+    key: str
+    connection_name: str
+    connection_id: str | None
+    managed_api_id: str | None
+    authentication: str | None
+    source: str
+
+    @property
+    def managed_identity(self) -> bool:
+        return self.authentication == "ManagedServiceIdentity"
+
+
+@dataclass(frozen=True)
+class ConnectionReference:
+    """A ``$connections`` key the definition uses, where, and whether it is wired.
+
+    ``wired`` is None for a bare definition: it carries no values, so whether the key
+    resolves cannot be known from the file, and False would claim a fault not shown.
+    """
+
+    workflow: str | None
+    key: str
+    used_by: tuple[str, ...]
+    wired: bool | None
+    source: str
+
+
+@dataclass(frozen=True)
+class DeployStep:
+    """A workflow's place in the deploy order: tier 0 first, then 1, and so on."""
+
+    workflow: str
+    tier: int
+    depends_on: tuple[str, ...]
+    source: str
+
+
+@dataclass(frozen=True)
+class Difference:
+    """One JSON path where two definitions differ."""
+
+    path: str
+    change: str  # changed, added (only in the second) or missing (only in the first)
+    reference: Any
+    difference: Any
+
+
+def parameter_status(
+    document: WorkflowDocument, supplied: Sequence[str] = ()
+) -> list[ParameterStatus]:
+    """One record per declared parameter: satisfied, and by which source.
+
+    A parameter is satisfied by being ``$connections`` (generated), by a name passed in
+    ``supplied`` (a deployment tool's own parameters input), by a ``defaultValue``, or by
+    a value in the wrapper (never for a secure type, nor for a bare definition, whose
+    ``parameters`` block is its declarations).
+    """
+    declarations = document.declarations or {}
+    values = document.parameter_values or {}
+    found = []
+    for name, declaration in declarations.items():
+        declaration = declaration if isinstance(declaration, Mapping) else {}
+        kind = declaration.get("type") if isinstance(declaration.get("type"), str) else None
+        secure = kind in SECURE_TYPES
+        in_wrapper = name in values
+        if name == CONNECTIONS:
+            verdict = (
+                True,
+                "generated",
+                "generated by the deployment tool from its connections input",
+            )
+        elif name in supplied:
+            verdict = (True, "supplied", "supplied out of band as a deployment tool input")
+        elif "defaultValue" in declaration:
+            verdict = (True, "default", "the declaration carries a defaultValue")
+        elif in_wrapper and not secure:
+            verdict = (True, "wrapper", f"the {document.shape} wrapper carries a value")
+        elif in_wrapper and secure:
+            verdict = (
+                False,
+                "none",
+                "secure type: the wrapper's value is ignored on purpose; supply it as an input "
+                "so the secret stays out of the definition",
+            )
+        elif document.shape == "bare":
+            verdict = (
+                False,
+                "none",
+                "no defaultValue, and a bare definition has no wrapper values",
+            )
+        else:
+            verdict = (
+                False,
+                "none",
+                f"no defaultValue, and the {document.shape} wrapper has no value",
+            )
+        satisfied, satisfied_by, reason = verdict
+        found.append(
+            ParameterStatus(
+                document.name, name, kind, secure, satisfied, satisfied_by, reason, document.source
+            )
+        )
+    return found
+
+
+def check(
+    document: WorkflowDocument,
+    *,
+    supplied: Sequence[str] = (),
+    connections: Sequence[str] = (),
+    callback_trigger: str | None = None,
+) -> list[Finding]:
+    """Every offline finding for ``document``, errors first.
+
+    Errors (the deploy is rejected): a parameter with no value; a supplied parameter the
+    definition does not declare; connections wired to a definition without a
+    ``$connections`` declaration; a callback trigger the definition does not have.
+    Warnings (it deploys, then surprises you): no triggers; no actions; a
+    ``$connections`` declaration with nothing wired to it.
+    """
+    findings: list[Finding] = []
+
+    def add(severity: str, rule: str, message: str) -> None:
+        findings.append(Finding(document.name, severity, rule, message, document.source))
+
+    for status in parameter_status(document, supplied):
+        if not status.satisfied:
+            add(
+                "error",
+                "parameter-has-no-value",
+                f"parameter {status.name!r} ({status.type}) has no value: {status.reason}",
+            )
+    declared = set(document.declarations or {})
+    for name in supplied:
+        if name not in declared:
+            add(
+                "error",
+                "supplied-parameter-not-declared",
+                f"parameter {name!r} is supplied but not declared: the definition is the "
+                "contract, and values only fill it",
+            )
+    if connections and CONNECTIONS not in declared:
+        add(
+            "error",
+            "connections-not-declared",
+            f"{len(connections)} connection(s) are wired, but the definition declares no "
+            "$connections parameter",
+        )
+    if not connections and CONNECTIONS in declared:
+        add(
+            "warning",
+            "connections-unwired",
+            "the definition declares $connections, but no connections are wired to it",
+        )
+    triggers = list(document.triggers)
+    if not triggers:
+        add("warning", "no-trigger", "the definition has no triggers, so nothing can start it")
+    if callback_trigger is not None and callback_trigger not in triggers:
+        add(
+            "error",
+            "callback-trigger-not-found",
+            f"callback trigger {callback_trigger!r} is not in the definition "
+            f"(triggers: {', '.join(triggers) or 'none'})",
+        )
+    if not document.actions:
+        add(
+            "warning",
+            "no-actions",
+            "the definition has no actions, so it does nothing when it runs",
+        )
+    return sorted(findings, key=lambda finding: finding.severity != "error")
+
+
+def connections_of(document: WorkflowDocument) -> list[Connection]:
+    """The connections a wrapper resolves; none for a bare definition, which holds none."""
+    entry = (document.parameter_values or {}).get(CONNECTIONS)
+    value = entry.get("value") if isinstance(entry, Mapping) else None
+    if not isinstance(value, Mapping):
+        return []
+    found = []
+    for key, detail in value.items():
+        detail = detail if isinstance(detail, Mapping) else {}
+        authentication = (detail.get("connectionProperties") or {}).get("authentication") or {}
+        kind = authentication.get("type") if isinstance(authentication, Mapping) else None
+        found.append(
+            Connection(
+                workflow=document.name,
+                key=key,
+                connection_name=str(detail.get("connectionName") or key),
+                connection_id=detail.get("connectionId"),
+                managed_api_id=detail.get("id"),
+                authentication=kind,
+                source=document.source,
+            )
+        )
+    return found
+
+
+def connection_references(document: WorkflowDocument) -> list[ConnectionReference]:
+    """Each ``$connections`` key the definition uses, from triggers and nested actions.
+
+    Triggers count: a Sentinel incident trigger reaches for a connection key exactly as an
+    action does.
+    """
+    wired: set[str] | None = None
+    if document.shape != "bare" and document.parameter_values is not None:
+        entry = document.parameter_values.get(CONNECTIONS)
+        value = entry.get("value") if isinstance(entry, Mapping) else None
+        if isinstance(value, Mapping):
+            wired = set(value)
+    nodes = [(f"triggers/{name}", trigger) for name, trigger in document.triggers.items()]
+    nodes += [(node.path, node.action) for node in action_nodes(document.actions)]
+    usage: dict[str, list[str]] = {}
+    for path, node in nodes:
+        inputs = node.get("inputs") if isinstance(node, Mapping) else None
+        if inputs is None:
+            continue
+        for key in _REFERENCE.findall(json.dumps(inputs)):
+            users = usage.setdefault(key, [])
+            if path not in users:
+                users.append(path)
+    return [
+        ConnectionReference(
+            document.name,
+            key,
+            tuple(users),
+            None if wired is None else key in wired,
+            document.source,
+        )
+        for key, users in usage.items()
+    ]
+
+
+def deploy_order(documents: Iterable[WorkflowDocument]) -> list[DeployStep]:
+    """The order a set of workflows must deploy in, from their dispatch actions.
+
+    Only a native Workflow dispatch action makes a dependency, since only its target is
+    resolved when the caller is written: a workflow that merely names a sibling (a
+    watchdog listing what it watches) does not. Tier 0 dispatches to nothing in the set;
+    each other workflow is one tier above the highest it calls. A cycle is an error.
+    """
+    by_name: dict[str, WorkflowDocument] = {}
+    for document in documents:
+        name = document.name or document.source
+        if name in by_name:
+            raise InputError(f"two definitions are both called {name!r}")
+        by_name[name] = document
+    depends: dict[str, tuple[str, ...]] = {}
+    for name, document in by_name.items():
+        targets = []
+        for node in action_nodes(document.actions):
+            if node.action.get("type") != "Workflow":
+                continue
+            workflow = ((node.action.get("inputs") or {}).get("host") or {}).get("workflow") or {}
+            if isinstance(workflow, Mapping) and isinstance(workflow.get("id"), str):
+                targets.append(workflow["id"].lower())
+        depends[name] = tuple(
+            sibling
+            for sibling in by_name
+            if sibling != name
+            and any(target.endswith(f"/workflows/{sibling.lower()}") for target in targets)
+        )
+    tiers: dict[str, int] = {}
+    remaining = list(by_name)
+    tier = 0
+    while remaining:
+        ready = [name for name in remaining if all(dep in tiers for dep in depends[name])]
+        if not ready:
+            raise InputError(
+                "no deploy order exists: these workflows dispatch to each other in a cycle: "
+                + ", ".join(sorted(remaining))
+            )
+        for name in ready:
+            tiers[name] = tier
+            remaining.remove(name)
+        tier += 1
+    return [
+        DeployStep(name, tiers[name], depends[name], by_name[name].source)
+        for name in sorted(by_name, key=lambda item: (tiers[item], item))
+    ]
+
+
+def compare(
+    reference: WorkflowDocument, difference: WorkflowDocument, *, parameter_values: bool = False
+) -> list[Difference]:
+    """The JSON paths where two definitions differ, after unwrapping both.
+
+    Unwrapping first means a code view export compares cleanly with an ARM GET of the
+    same workflow, and a rendered template with what is deployed: drift review as a diff.
+    """
+    found: list[Difference] = []
+    _walk(reference.definition, difference.definition, "definition", found)
+    if parameter_values:
+        _walk(reference.parameter_values, difference.parameter_values, "parameters", found)
+    return found
+
+
+def _walk(left: Any, right: Any, path: str, found: list[Difference]) -> None:
+    if left is None and right is None:
+        return
+    if left is None or right is None:
+        change = "added" if left is None else "missing"
+        found.append(Difference(path, change, left, right))
+        return
+    if isinstance(left, Mapping) and isinstance(right, Mapping):
+        for key in sorted(set(left) | set(right)):
+            _walk(left.get(key), right.get(key), f"{path}.{key}", found)
+        return
+    if isinstance(left, list) and isinstance(right, list):
+        for index in range(max(len(left), len(right))):
+            _walk(
+                left[index] if index < len(left) else None,
+                right[index] if index < len(right) else None,
+                f"{path}[{index}]",
+                found,
+            )
+        return
+    if json.dumps(left, sort_keys=True) != json.dumps(right, sort_keys=True):
+        found.append(Difference(path, "changed", left, right))
+
+
+def with_parameter_defaults(
+    document: WorkflowDocument, *, force: bool = False
+) -> tuple[dict[str, Any], int]:
+    """The definition with each wrapper value copied onto its declaration as a default.
+
+    That makes an export deployable on its own. ``$connections`` is left for the
+    deployment tool to generate, and secure types are never copied: writing a secret into
+    the definition is what they exist to avoid. The result then carries the SOURCE
+    estate's values; ``rewrite_references`` re-points them. Returns the new definition
+    and how many defaults were added.
+    """
+    if document.shape == "bare":
+        raise InputError(
+            f"{document.source} is a bare definition: it has declarations but no values to copy"
+        )
+    definition = copy.deepcopy(dict(document.definition))
+    declarations = definition.get("parameters")
+    values = document.parameter_values or {}
+    added = 0
+    if isinstance(declarations, dict):
+        for name, declaration in declarations.items():
+            if name == CONNECTIONS or not isinstance(declaration, dict):
+                continue
+            if declaration.get("type") in SECURE_TYPES:
+                continue
+            if "defaultValue" in declaration and not force:
+                continue
+            entry = values.get(name)
+            if isinstance(entry, Mapping) and "value" in entry:
+                declaration["defaultValue"] = copy.deepcopy(entry["value"])
+                added += 1
+    return definition, added
+
+
+def rewrite_references(text: str, replacements: Sequence[tuple[str, str]]) -> str:
+    """``text`` with each literal find and replace applied, in the order given.
+
+    Estate-specific references (subscription ids, resource group and connection names,
+    instance URLs) live in the serialised JSON, so rewriting the text keeps everything
+    else byte-identical. Put the most specific replacement first: a later, general one
+    would otherwise rewrite what an earlier one produced.
+    """
+    for old, new in replacements:
+        if not old:
+            raise InputError("a replacement needs something to find")
+        text = text.replace(old, new)
+    return text