temporal-flowrensic-python-sdk 0.6.63__py3-none-any.whl

temporal_flowrensic/_names.py

@@ -0,0 +1,42 @@
+"""The names of workflow interceptors, as Flowrensic is sent them by a worker
+and by a replay, so the two lists can be compared."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Optional, Sequence
+
+from temporal_flowrensic._gen.flowrensic.v1.replay_pb2 import WorkflowInterceptors
+
+__all__ = ["interceptor_list", "interceptor_name"]
+
+_EXTENSIONS = re.compile(r"\.(?:[cm]?[jt]s|[jt]sx|py)$")
+
+
+def interceptor_name(interceptor: Any) -> str:
+    """The name of a workflow interceptor. A path or module name is named by
+    its file name without the folder and the extension, and a file named
+    index or __init__ by the name of its folder, as the Node SDK names a
+    module. So /app/interceptors/metrics.py, src/interceptors/metrics.py and
+    app.interceptors.metrics are all metrics. A class is named by its class
+    name, and any other object by the name of its class."""
+    if isinstance(interceptor, str):
+        parts = [part for part in re.split(r"[/\\]", interceptor) if part]
+        file = _EXTENSIONS.sub("", parts[-1] if parts else "")
+        if "/" not in interceptor and "\\" not in interceptor and "." in file:
+            dotted = [part for part in file.split(".") if part]
+            parts = dotted
+            file = dotted[-1] if dotted else ""
+        if file in ("index", "__init__") and len(parts) > 1:
+            return _EXTENSIONS.sub("", parts[-2])
+        return file
+    if isinstance(interceptor, type):
+        return interceptor.__name__
+    return type(interceptor).__name__
+
+
+def interceptor_list(interceptors: Optional[Sequence[Any]]) -> Optional[WorkflowInterceptors]:
+    """The interceptors as Flowrensic is sent them, in their order. None sends no list."""
+    if interceptors is None:
+        return None
+    return WorkflowInterceptors(names=[interceptor_name(one) for one in interceptors])

temporal_flowrensic/_queries.py

@@ -0,0 +1,61 @@
+"""The buffer of the queries the workflows of a worker answered, counted per
+workflow type, query name and minute when a batch is sent."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from temporal_flowrensic._gen.flowrensic.v1.client_calls_pb2 import IngestQueryCountsRequest, QueryCount
+from temporal_flowrensic._sink import BufferTarget, SendId, client_headers, ingest_client_sync
+
+__all__ = ["Answered", "QueryTarget"]
+
+_MINUTE = 60
+
+
+@dataclass(frozen=True)
+class Answered:
+    """One query a workflow answered, ok when its handler returned, at a time in seconds."""
+
+    namespace: str
+    workflow_type: str
+    name: str
+    ok: bool
+    at: float
+
+
+class QueryTarget(BufferTarget[Answered]):
+    def __init__(
+        self, server_base_url: str, env: str, env_access_key: str, ephemeral_run: str
+    ) -> None:
+        self._ingest = ingest_client_sync(server_base_url)
+        self._env = env
+        self._headers = client_headers(env, env_access_key, ephemeral_run)
+
+    def send(self, records: list[Answered], timeout_ms: int, id: SendId) -> None:
+        counts: dict[tuple[str, str, str, int], QueryCount] = {}
+        for one in records:
+            minute = int(one.at // _MINUTE) * _MINUTE
+            key = (one.namespace, one.workflow_type, one.name, minute)
+            count = counts.get(key)
+            if count is None:
+                count = QueryCount(
+                    namespace=one.namespace,
+                    env=self._env,
+                    workflow_type=one.workflow_type,
+                    query_name=one.name,
+                )
+                count.minute.FromSeconds(minute)
+                counts[key] = count
+            if one.ok:
+                count.answered += 1
+            else:
+                count.failed += 1
+        self._ingest.ingest_query_counts(
+            IngestQueryCountsRequest(counts=list(counts.values()), sender=id.sender, send_number=id.send_number),
+            headers=self._headers,
+            timeout_ms=max(1, timeout_ms),
+        )
+
+    def env_of(self, record: Answered) -> str:
+        return self._env

temporal_flowrensic/_replay.py

@@ -0,0 +1,378 @@
+"""Replay before deploy: fetch the replay set Flowrensic picks for a workflow
+type in each of the envs named, write it to disk the way the CLI and the Node SDK do, and send
+the verdicts of a replay back. Nothing here runs a workflow; the CI job
+replays with temporalio's Replayer."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import sys
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Optional, Sequence
+
+from connectrpc.code import Code
+from connectrpc.errors import ConnectError
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from temporal_flowrensic._ci import ci_error, ci_failure
+from temporal_flowrensic._gen.flowrensic.v1.replay_pb2 import (
+    RUN_STATUS_CANCELED,
+    RUN_STATUS_COMPLETED,
+    RUN_STATUS_CONTINUED_AS_NEW,
+    RUN_STATUS_FAILED,
+    RUN_STATUS_PAUSED,
+    RUN_STATUS_RUNNING,
+    RUN_STATUS_TERMINATED,
+    RUN_STATUS_TIMED_OUT,
+    RUN_STATUS_UNSPECIFIED,
+    IngestReplayResultsRequest,
+    ListReplayHistoriesRequest,
+    ListReplayHistoriesResponse,
+    ReplayResult,
+)
+from temporal_flowrensic._gen.flowrensic.v1.page_pb2 import PageRequest
+from temporal_flowrensic._gen.flowrensic.v1.services_connect import IngestServiceClient, QueryServiceClient
+from temporal_flowrensic._names import interceptor_list
+from temporal_flowrensic._sink import client_headers
+
+__all__ = [
+    "INDEX_NAME",
+    "LeftOutEntry",
+    "ReplaySet",
+    "ReplaySetEntry",
+    "ReplayVerdict",
+    "fetch_replay_set",
+    "list_history_files",
+    "publish_verdicts",
+    "read_replay_set",
+]
+
+INDEX_NAME = "replay-set.json"
+"""The manifest written beside the histories, the same one the CLI writes."""
+
+_DOWNLOAD_TIMEOUT_MS = 60_000
+_PUBLISH_TIMEOUT_MS = 30_000
+_SEND_AGAIN_FOR = 600.0
+
+# A manifest reads completed, the same label the CLI writes.
+_STATUS_LABELS = {
+    RUN_STATUS_UNSPECIFIED: "unspecified",
+    RUN_STATUS_RUNNING: "running",
+    RUN_STATUS_COMPLETED: "completed",
+    RUN_STATUS_FAILED: "failed",
+    RUN_STATUS_CANCELED: "canceled",
+    RUN_STATUS_TERMINATED: "terminated",
+    RUN_STATUS_TIMED_OUT: "timed_out",
+    RUN_STATUS_CONTINUED_AS_NEW: "continued_as_new",
+    RUN_STATUS_PAUSED: "paused",
+}
+
+
+def _iso(ts: Optional[Timestamp]) -> str:
+    seconds = 0.0 if ts is None else ts.seconds + ts.nanos / 1e9
+    return datetime.fromtimestamp(seconds, tz=timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _now() -> str:
+    return datetime.now(tz=timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+@dataclass(frozen=True)
+class ReplaySetEntry:
+    """One history on disk, with what the server chose it by."""
+
+    run_id: str
+    file: str
+    status: str
+    code_versions: list[str]
+    events: int
+    started_at: str
+    env: Optional[str] = None
+    closed_at: Optional[str] = None
+
+    def to_json(self) -> dict[str, Any]:
+        out: dict[str, Any] = {"run_id": self.run_id, "file": self.file, "status": self.status}
+        if self.env is not None:
+            out["env"] = self.env
+        out["code_versions"] = list(self.code_versions)
+        out["events"] = self.events
+        out["started_at"] = self.started_at
+        if self.closed_at is not None:
+            out["closed_at"] = self.closed_at
+        return out
+
+
+@dataclass(frozen=True)
+class LeftOutEntry:
+    """A stored run the server could not read, so it is not in the set."""
+
+    run_id: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class ReplaySet:
+    """The set of one workflow type in the envs named, as the manifest holds it."""
+
+    workflow_type: str
+    envs: list[str]
+    """The envs the histories were fetched from, in the order they were named."""
+    fetched_at: str
+    server: str
+    histories: list[ReplaySetEntry]
+    left_out: list[LeftOutEntry] = field(default_factory=list)
+    failed: Optional[str] = None
+    """Why the download failed, when fail_on_error was False and it did. The set is then empty."""
+
+    def to_json(self) -> dict[str, Any]:
+        """The manifest as the CLI and the Node SDK write it."""
+        out: dict[str, Any] = {"workflow_type": self.workflow_type, "envs": list(self.envs)}
+        out["fetched_at"] = self.fetched_at
+        out["server"] = self.server
+        out["histories"] = [one.to_json() for one in self.histories]
+        out["left_out"] = [{"run_id": one.run_id, "reason": one.reason} for one in self.left_out]
+        if self.failed is not None:
+            out["failed"] = self.failed
+        return out
+
+    @staticmethod
+    def from_json(value: Any) -> ReplaySet:
+        if not isinstance(value, dict) or not isinstance(value.get("workflow_type"), str):
+            raise ValueError("not a replay set manifest")
+        if not isinstance(value.get("histories"), list):
+            raise ValueError("not a replay set manifest")
+        return ReplaySet(
+            workflow_type=value["workflow_type"],
+            envs=[str(one) for one in value.get("envs", [])],
+            fetched_at=value.get("fetched_at", ""),
+            server=value.get("server", ""),
+            histories=[
+                ReplaySetEntry(
+                    run_id=one["run_id"],
+                    file=one["file"],
+                    status=one.get("status", ""),
+                    env=one.get("env"),
+                    code_versions=list(one.get("code_versions", [])),
+                    events=one.get("events", 0),
+                    started_at=one.get("started_at", ""),
+                    closed_at=one.get("closed_at"),
+                )
+                for one in value["histories"]
+            ],
+            left_out=[LeftOutEntry(one["run_id"], one.get("reason", "")) for one in value.get("left_out", [])],
+            failed=value.get("failed"),
+        )
+
+
+@dataclass(frozen=True)
+class ReplayVerdict:
+    """What one history did when replayed against the code under test."""
+
+    run_id: str
+    workflow_type: str
+    passed: bool
+    error: Optional[str] = None
+    """The replayer's own message, kept as it is: it names what diverged."""
+
+
+def _write(path: str, data: bytes) -> None:
+    with open(path, "wb") as f:
+        f.write(data)
+    os.chmod(path, 0o600)
+
+
+async def fetch_replay_set(
+    *,
+    server_base_url: str,
+    replay_history_workflow: str,
+    replay_history_out_dir: str,
+    replay_history_from_envs: Sequence[str],
+    current_env: str,
+    env_access_key: str,
+    current_ephemeral_run: str = "",
+    timeout_ms: int = _DOWNLOAD_TIMEOUT_MS,
+    fail_on_error: bool = True,
+) -> ReplaySet:
+    """Fetches the set the server picks for a workflow type in each env named,
+    in one request, and writes each history to replay_history_out_dir as
+    RUN_ID.json, as the server serves it, and the manifest replay-set.json
+    beside them.
+
+    replay_history_from_envs is a list of env names, such as ["prod",
+    "staging"]. Each must be current_env or an env it may read. A name given
+    twice is read once. A plain string raises ValueError. current_env,
+    env_access_key and current_ephemeral_run are those of the caller and go in
+    their headers.
+
+    The download has a time limit of timeout_ms milliseconds, 60000 by
+    default. When it passes or
+    Flowrensic refuses, it raises CIError with a message that says Flowrensic
+    could not be reached or what it answered. With fail_on_error False it
+    returns an empty set whose failed field says why, and writes nothing."""
+    if isinstance(replay_history_from_envs, str):
+        raise ValueError("flowrensic: replay_history_from_envs takes a list of env names, got a plain string")
+    workflow_type = replay_history_workflow
+    envs = list(dict.fromkeys(replay_history_from_envs))
+    dir = replay_history_out_dir
+    try:
+        async with QueryServiceClient(server_base_url.rstrip("/"), send_compression=None) as query:
+            # The answer comes in pages of whole env sets: ask until the server
+            # says there is no more, and keep every page in order.
+            found = ListReplayHistoriesResponse()
+            page_token = ""
+            while True:
+                page = await query.list_replay_histories(
+                    ListReplayHistoriesRequest(
+                        workflow_type=workflow_type, envs=envs, page=PageRequest(page_token=page_token)
+                    ),
+                    headers=client_headers(current_env, env_access_key, current_ephemeral_run),
+                    timeout_ms=max(1, timeout_ms),
+                )
+                found.histories.extend(page.histories)
+                found.left_out.extend(page.left_out)
+                page_token = page.page.next_page_token
+                if not page_token:
+                    break
+    except Exception as error:  # noqa: BLE001
+        if fail_on_error:
+            raise ci_error("replay set download", server_base_url, timeout_ms, error) from error
+        return ReplaySet(
+            workflow_type=workflow_type,
+            envs=envs,
+            fetched_at=_now(),
+            server=server_base_url,
+            histories=[],
+            failed=ci_failure("replay set download", server_base_url, timeout_ms, error),
+        )
+
+    os.makedirs(dir, mode=0o750, exist_ok=True)
+    histories = []
+    for history in found.histories:
+        file = f"{history.run_id}.json"
+        # Written and renamed, so a failed write never leaves half a history.
+        temp = os.path.join(dir, f".fetch-{history.run_id}")
+        _write(temp, history.history)
+        os.replace(temp, os.path.join(dir, file))
+        histories.append(
+            ReplaySetEntry(
+                run_id=history.run_id,
+                file=file,
+                status=_STATUS_LABELS.get(history.status, "unspecified"),
+                env=history.env or None,
+                code_versions=list(history.code_versions),
+                events=history.events,
+                started_at=_iso(history.started_at if history.HasField("started_at") else None),
+                closed_at=_iso(history.closed_at) if history.HasField("closed_at") else None,
+            )
+        )
+    found_set = ReplaySet(
+        workflow_type=workflow_type,
+        envs=envs,
+        fetched_at=_now(),
+        server=server_base_url,
+        histories=histories,
+        left_out=[LeftOutEntry(one.run_id, one.reason) for one in found.left_out],
+    )
+    _write(os.path.join(dir, INDEX_NAME), (json.dumps(found_set.to_json(), indent=2) + "\n").encode("utf-8"))
+    _log_fetched(found_set, len(found.histories), dir)
+    return found_set
+
+
+def _log_fetched(found_set: ReplaySet, got: int, dir: str) -> None:
+    """Writes to stderr what a download got, the same lines in every SDK and
+    the CLI: the workflow type and the envs, how many histories the answer held
+    and how many were saved, the run id of each, and each run left out with its
+    reason."""
+    lines = [
+        f"replay set: workflow {found_set.workflow_type}, envs {', '.join(found_set.envs)}",
+        f"got {got} histories, saved {len(found_set.histories)} in {dir}",
+        *(f"  {one.run_id}" for one in found_set.histories),
+        *(f"left out {one.run_id}: {one.reason}" for one in found_set.left_out),
+    ]
+    sys.stderr.write("\n".join(lines) + "\n")
+    sys.stderr.flush()
+
+
+def read_replay_set(dir: str) -> ReplaySet:
+    """Reads a manifest a fetch left behind, the CLI's or this package's."""
+    path = os.path.join(dir, INDEX_NAME)
+    with open(path, encoding="utf-8") as f:
+        try:
+            return ReplaySet.from_json(json.load(f))
+        except ValueError as error:
+            raise ValueError(f"{path} is not a replay set manifest") from error
+
+
+def list_history_files(dir: str) -> list[str]:
+    """The histories in a directory, without the manifest."""
+    return sorted(name for name in os.listdir(dir) if name.endswith(".json") and name != INDEX_NAME)
+
+
+def _wait_before_sending_again(error: BaseException) -> Optional[float]:
+    """Seconds to wait before sending again a call that Flowrensic did not
+    answer or answered try again later, one second. None for any other error,
+    which is not sent again."""
+    if not isinstance(error, ConnectError):
+        return None
+    if error.code not in (Code.UNAVAILABLE, Code.DEADLINE_EXCEEDED, Code.RESOURCE_EXHAUSTED):
+        return None
+    return 1.0
+
+
+async def publish_verdicts(
+    *,
+    server_base_url: str,
+    code_version: str,
+    verdicts: Sequence[ReplayVerdict],
+    current_env: str,
+    env_access_key: str,
+    current_ephemeral_run: str = "",
+    workflow_interceptors: Optional[Sequence[Any]] = None,
+    send_again_for: float = _SEND_AGAIN_FOR,
+    timeout_ms: int = _PUBLISH_TIMEOUT_MS,
+) -> None:
+    """Reports what a replay found, against the history and the code version
+    replayed, so a break is put down to a code change. workflow_interceptors
+    are the workflow interceptors the replay ran with, sent by name; left out,
+    no list is sent. A call Flowrensic does not answer, or answers try again
+    later, is sent again for up to send_again_for seconds, 10 minutes by
+    default. Each call has a time limit of timeout_ms milliseconds, 30000 by
+    default. When it gives up or Flowrensic refuses, it raises CIError.
+    current_env, env_access_key and current_ephemeral_run go in their headers."""
+    if not verdicts:
+        return
+    interceptors = interceptor_list(workflow_interceptors)
+    replayed_at = Timestamp()
+    replayed_at.FromNanoseconds(int(time.time() * 1_000_000_000))
+    results = []
+    for v in verdicts:
+        one = ReplayResult(
+            run_id=v.run_id,
+            code_version=code_version,
+            workflow_type=v.workflow_type,
+            passed=v.passed,
+            error=v.error or "",
+            replayed_at=replayed_at,
+        )
+        if interceptors is not None:
+            one.workflow_interceptors.CopyFrom(interceptors)
+        results.append(one)
+    give_up_at = time.monotonic() + send_again_for
+    async with IngestServiceClient(server_base_url.rstrip("/"), send_compression=None) as ingest:
+        while True:
+            try:
+                await ingest.ingest_replay_results(
+                    IngestReplayResultsRequest(results=results),
+                    headers=client_headers(current_env, env_access_key, current_ephemeral_run),
+                    timeout_ms=max(1, timeout_ms),
+                )
+                return
+            except Exception as error:  # noqa: BLE001
+                wait = _wait_before_sending_again(error)
+                if wait is None or time.monotonic() + wait > give_up_at:
+                    raise ci_error("replay results", server_base_url, timeout_ms, error) from error
+                await asyncio.sleep(wait)