concurim 0.1.0__py3-none-any.whl

concurim/__init__.py

@@ -0,0 +1,72 @@
+"""Concurim: see how Python runs together."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from concurim.config import RecorderConfig
+from concurim.instrumentation.aio_events import (
+    TracedAsyncCondition,
+    TracedAsyncEvent,
+)
+from concurim.instrumentation.aio_primitives import (
+    TracedAsyncBoundedSemaphore,
+    TracedAsyncLock,
+    TracedAsyncQueue,
+    TracedAsyncSemaphore,
+)
+from concurim.instrumentation.barriers import (
+    TracedAsyncBarrier,
+    TracedBarrier,
+)
+from concurim.instrumentation.executors import TracedThreadPoolExecutor
+from concurim.instrumentation.mp_pool import TracedPool
+from concurim.instrumentation.mp_process import TracedProcess
+from concurim.instrumentation.process_pool import TracedProcessPoolExecutor
+from concurim.instrumentation.queues import QueueItem, TracedQueue
+from concurim.instrumentation.sync import (
+    TracedBoundedSemaphore,
+    TracedLock,
+    TracedRLock,
+    TracedSemaphore,
+)
+from concurim.instrumentation.sync_events import (
+    TracedCondition,
+    TracedEvent,
+)
+from concurim.instrumentation.threads import TracedThread
+from concurim.instrumentation.work import work
+from concurim.recorder.lifecycle import active_recorder, record
+from concurim.recorder.serialization import load_trace
+from concurim.replay.engine import build_replay
+
+__all__ = [
+    "QueueItem",
+    "RecorderConfig",
+    "TracedAsyncBarrier",
+    "TracedAsyncBoundedSemaphore",
+    "TracedAsyncCondition",
+    "TracedAsyncEvent",
+    "TracedAsyncLock",
+    "TracedAsyncQueue",
+    "TracedAsyncSemaphore",
+    "TracedBarrier",
+    "TracedBoundedSemaphore",
+    "TracedCondition",
+    "TracedEvent",
+    "TracedLock",
+    "TracedQueue",
+    "TracedRLock",
+    "TracedSemaphore",
+    "TracedThread",
+    "TracedPool",
+    "TracedProcess",
+    "TracedProcessPoolExecutor",
+    "TracedThreadPoolExecutor",
+    "__version__",
+    "active_recorder",
+    "build_replay",
+    "load_trace",
+    "record",
+    "work",
+]

concurim/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m concurim``."""
+
+import sys
+
+from concurim.cli import main
+
+sys.exit(main())

concurim/cli.py

@@ -0,0 +1,262 @@
+"""Command-line interface: record, view, demo, inspect."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+import webbrowser
+from collections.abc import Sequence
+from pathlib import Path
+
+from concurim import __version__
+from concurim.config import RecorderConfig
+from concurim.export import cli as export_cli
+from concurim.instrumentation.auto import LEVELS
+from concurim.program_check import ProgramError
+from concurim.recorder.events import TraceFormatError
+from concurim.recorder.serialization import Trace, load_trace
+from concurim.samples import sample_file
+
+SHARING_WARNING = (
+    "Note: traces can contain file paths, thread names and (if enabled) "
+    "source code or payload previews. Review a trace before sharing it."
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="concurim", description="See how Python runs together."
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"concurim {__version__}"
+    )
+    _add_server_options(parser)
+    parser.set_defaults(func=_cmd_home)
+    sub = parser.add_subparsers(dest="command")
+
+    rec = sub.add_parser(
+        "record",
+        help="run a script and record instrumented activity",
+        description="Run SCRIPT and record activity on Concurim-instrumented "
+        "objects. Options must come before SCRIPT; everything after SCRIPT "
+        "is passed to it unchanged.",
+    )
+    rec.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="trace path (default: <script>.concurim)",
+    )
+    rec.add_argument(
+        "--include-source",
+        action="store_true",
+        help="embed project source files in the trace",
+    )
+    rec.add_argument(
+        "--preview",
+        action="store_true",
+        help="record bounded previews of primitive payloads",
+    )
+    rec.add_argument(
+        "--auto",
+        choices=LEVELS,
+        default="threads",
+        help="automatic instrumentation: none (only Concurim objects), "
+        "threads (default), queues (threads + queue.Queue)",
+    )
+    rec.add_argument("--max-events", type=int, default=1_000_000)
+    rec.add_argument(
+        "--light",
+        action="store_true",
+        help="lightweight mode: no source locations",
+    )
+    rec.add_argument(
+        "--open",
+        action="store_true",
+        help="open the viewer when the recording finishes",
+    )
+    _add_server_options(rec)
+    rec.add_argument("script", type=Path)
+    rec.add_argument("script_args", nargs=argparse.REMAINDER)
+    rec.set_defaults(func=_cmd_record)
+
+    for name, func, help_ in (
+        ("view", _cmd_view, "open a recorded trace"),
+        ("compare", _cmd_compare, "show two recordings side by side"),
+        ("demo", _cmd_demo, "record and view a demo"),
+    ):
+        cmd = sub.add_parser(name, help=help_)
+        if name == "view":
+            cmd.add_argument("trace", type=Path)
+        if name == "compare":
+            cmd.add_argument("first", type=Path)
+            cmd.add_argument("second", type=Path)
+        _add_server_options(cmd)
+        cmd.set_defaults(func=func)
+
+    export_cli.add_parser(sub)
+    insp = sub.add_parser("inspect", help="print trace metadata")
+    insp.add_argument("trace", type=Path)
+    insp.set_defaults(func=_cmd_inspect)
+    return parser
+
+
+def _add_server_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=0,
+        help="port to serve on (default: any free port)",
+    )
+    parser.add_argument("--no-browser", action="store_true")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """CLI entry point; returns the process exit status."""
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except (TraceFormatError, FileNotFoundError, ProgramError) as error:
+        print(f"concurim: error: {error}", file=sys.stderr)
+        return 2
+
+
+def _cmd_record(args: argparse.Namespace) -> int:
+    from concurim.runner import run_recorded
+
+    script: Path = args.script
+    if not script.is_file():
+        raise FileNotFoundError(f"script not found: {script}")
+    output = args.output or Path(f"{script.stem}.concurim")
+    config = RecorderConfig(
+        max_events=args.max_events,
+        include_source=args.include_source,
+        preview=args.preview,
+        capture_source_locations=not args.light,
+        auto=args.auto,
+        # Paths are shown relative to the script, wherever it is run from.
+        source_root=str(script.resolve().parent),
+    )
+    code, count = run_recorded(
+        script, args.script_args, output, config, view_on_stop=args.open
+    )
+    print(f"concurim: recorded {count} events to {output}", file=sys.stderr)
+    print(f"concurim: view it with: concurim view {output}", file=sys.stderr)
+    print(SHARING_WARNING, file=sys.stderr)
+    stuck = _still_blocked(output)
+    if args.open and output.is_file():
+        _serve(
+            load_trace(output),
+            args.port,
+            not args.no_browser,
+            output.name,
+        )
+    if stuck:
+        # Non-daemon threads are still blocked. Returning would hang in
+        # interpreter shutdown and never give the terminal back.
+        os._exit(code)
+    return code
+
+
+def _serve(
+    trace: Trace | None,
+    port: int,
+    open_browser: bool,
+    label: str = "trace",
+    compare: tuple[tuple[Trace, str], ...] = (),
+) -> int:
+    from concurim.viewer.server import ViewerServer
+
+    if not (Path(__file__).parent / "viewer/static/index.html").is_file():
+        raise FileNotFoundError("viewer assets missing (run scripts/build.sh)")
+    server = ViewerServer(trace, port, label=label, compare=compare)
+    url = server.url + ("?compare=1" if compare else "")
+    print(f"concurim: viewer at {url} (Ctrl-C to stop)")
+    if open_browser:
+        webbrowser.open(url)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nconcurim: stopped")
+    finally:
+        server.server_close()
+    return 0
+
+
+def _cmd_home(args: argparse.Namespace) -> int:
+    return _serve(None, args.port, not args.no_browser)
+
+
+def _cmd_view(args: argparse.Namespace) -> int:
+    return _serve(
+        load_trace(args.trace), args.port, not args.no_browser, args.trace.name
+    )
+
+
+def _cmd_compare(args: argparse.Namespace) -> int:
+    pair = tuple((load_trace(p), p.name) for p in (args.first, args.second))
+    return _serve(None, args.port, not args.no_browser, compare=pair)
+
+
+def _cmd_demo(args: argparse.Namespace) -> int:
+    from concurim.runner import run_recorded
+
+    script = sample_file("threaded_pipeline.py")
+    out = Path(tempfile.mkdtemp(prefix="concurim-demo-")) / "demo.concurim"
+    # The bundled example is trusted, so its source is embedded.
+    config = RecorderConfig(include_source=True, source_root=str(script.parent))
+    code, count = run_recorded(script, [], out, config)
+    print(f"concurim: recorded {count} events from {script.name}")
+    if code != 0:
+        return code
+    return _serve(load_trace(out), args.port, not args.no_browser, script.name)
+
+
+def _still_blocked(path: Path) -> bool:
+    """Whether the written trace says other threads were still running."""
+    if not path.is_file():
+        return False
+    try:
+        program = load_trace(path).metadata.get("program") or {}
+    except (TraceFormatError, OSError):
+        return False
+    return bool(program.get("still_blocked"))
+
+
+def _cmd_inspect(args: argparse.Namespace) -> int:
+    trace = load_trace(args.trace)
+    meta = trace.metadata
+    env = meta.get("environment", {})
+    print(f"trace:          {args.trace}")
+    print(
+        f"concurim:       {meta.get('concurim_version')} "
+        f"(format {meta.get('format_version')})"
+    )
+    print(f"recorded:       {meta.get('created_at_utc')}")
+    print(f"duration:       {meta.get('duration_ns', 0) / 1e9:.3f}s")
+    print(f"events:         {len(trace.events)}")
+    print(
+        f"python:         {env.get('implementation')} "
+        f"{env.get('python_version')} on {env.get('platform')} "
+        f"(free-threaded build: {env.get('free_threaded_build')}, "
+        f"GIL enabled: {env.get('gil_enabled')})"
+    )
+    print(f"sources:        {len(trace.sources)} embedded")
+    program = meta.get("program") or {}
+    if program.get("outcome"):
+        print(f"outcome:        {program['outcome']}")
+    if program.get("still_blocked"):
+        print("WARNING:        stopped while other threads were still running")
+    if meta.get("truncated"):
+        print(
+            f"WARNING:        truncated, {meta.get('dropped_events')} "
+            "events dropped (buffer limit)"
+        )
+    coverage = meta.get("coverage", {})
+    for key in ("observed", "not_observed"):
+        print(f"{key.replace('_', ' ')}:")
+        for line in coverage.get(key, []):
+            print(f"  - {line}")
+    return 0

concurim/config.py

@@ -0,0 +1,81 @@
+"""Recorder configuration."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# Automatic instrumentation levels; each includes the ones before it.
+LEVELS = ("none", "threads", "queues", "sync", "asyncio")
+
+
+@dataclass(frozen=True, slots=True)
+class RecorderConfig:
+    """Options controlling what is recorded.
+
+    Attributes:
+        max_events: Hard cap on buffered events. When reached, *new* events
+            are dropped (the earliest events keep the trace coherent) and the
+            trace is marked truncated.
+        include_source: Embed the text of referenced source files in the
+            trace. Off by default because sources may be sensitive.
+        capture_source_locations: Record file/line/function per event. Turn
+            off for the lightest recording mode.
+        preview: Record a bounded preview of primitive payloads. Off by
+            default because payloads may be sensitive.
+        preview_max_chars: Maximum characters kept in a preview.
+        redact_patterns: Regexes replaced by ``[redacted]`` in previews.
+        auto: Automatic instrumentation level while recording, each level
+            including the previous: ``"none"`` (only Concurim-managed
+            objects), ``"threads"`` (every ``threading.Thread``),
+            ``"queues"`` (``queue.Queue`` objects created during the run),
+            ``"sync"`` (locks, semaphores and ``ThreadPoolExecutor``
+            created by your code) or ``"asyncio"`` (tasks and asyncio
+            queues, locks and semaphores).
+        source_root: Directory that recorded paths are shown relative to,
+            and the only place source files may be embedded from. Defaults
+            to the current directory when recording starts.
+        sample_interval_ms: How often (milliseconds) a background thread
+            samples which line each worker is running, so the code view can
+            follow the work between instrumented calls. Only used with
+            ``include_source``; ``0`` turns sampling off. Sampling is
+            approximate and slightly slows the program.
+    """
+
+    max_events: int = 1_000_000
+    include_source: bool = False
+    capture_source_locations: bool = True
+    preview: bool = False
+    preview_max_chars: int = 48
+    redact_patterns: tuple[str, ...] = ()
+    source_root: str | None = None
+    auto: str = "none"
+    sample_interval_ms: float = 5.0
+
+    def __post_init__(self) -> None:
+        if self.auto not in LEVELS:
+            raise ValueError(f"unknown auto level {self.auto!r}")
+        if self.max_events < 1:
+            raise ValueError("max_events must be positive")
+        if not 0 <= self.preview_max_chars <= 256:
+            raise ValueError("preview_max_chars must be in [0, 256]")
+        if not 0 <= self.sample_interval_ms <= 10_000:
+            raise ValueError("sample_interval_ms must be in [0, 10000]")
+        for pattern in self.redact_patterns:
+            re.compile(pattern)
+
+    @classmethod
+    def light(cls) -> RecorderConfig:
+        """Lightweight mode: no source locations, no previews."""
+        return cls(capture_source_locations=False)
+
+    def public_dict(self) -> dict[str, object]:
+        """Config values safe to store in trace metadata."""
+        return {
+            "max_events": self.max_events,
+            "include_source": self.include_source,
+            "capture_source_locations": self.capture_source_locations,
+            "preview": self.preview,
+            "auto": self.auto,
+            "sample_interval_ms": self.sample_interval_ms,
+        }

concurim/export/__init__.py

@@ -0,0 +1 @@
+"""Render recordings as video, GIF or PNG frames (see ``video.py``)."""

concurim/export/cdp.py

@@ -0,0 +1,224 @@
+"""A tiny Chrome DevTools Protocol client over a pipe (standard library).
+
+Chrome-family browsers started with ``--remote-debugging-pipe`` read JSON
+commands from file descriptor 3 and answer on file descriptor 4, each
+message ending in a NUL byte. That is all the exporter needs, so no browser
+automation package is required. POSIX only (macOS and Linux).
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import json
+import os
+import select
+import shutil
+import signal
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+BROWSER_ENV = "CONCURIM_BROWSER"
+_MAC_APPS = (
+    "Google Chrome.app/Contents/MacOS/Google Chrome",
+    "Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+    "Chromium.app/Contents/MacOS/Chromium",
+    "Brave Browser.app/Contents/MacOS/Brave Browser",
+)
+_PATH_NAMES = (
+    "google-chrome",
+    "google-chrome-stable",
+    "chromium",
+    "chromium-browser",
+    "microsoft-edge",
+    "microsoft-edge-stable",
+    "brave-browser",
+)
+_TIMEOUT_S = 30.0
+
+
+class ExportError(Exception):
+    """A problem the user can fix (missing browser, missing ffmpeg, ...)."""
+
+
+def find_browser() -> str:
+    """Path of a Chrome-family browser.
+
+    ``$CONCURIM_BROWSER`` wins; otherwise common install locations and
+    ``PATH`` are searched. Raises ``ExportError`` with advice if none exists.
+    """
+    wanted = os.environ.get(BROWSER_ENV)
+    if wanted:
+        if shutil.which(wanted) or Path(wanted).is_file():
+            return shutil.which(wanted) or wanted
+        raise ExportError(f"{BROWSER_ENV}={wanted!r} is not an executable")
+    for app in _MAC_APPS:
+        for base in (Path("/Applications"), Path.home() / "Applications"):
+            if (base / app).is_file():
+                return str(base / app)
+    for name in _PATH_NAMES:
+        found = shutil.which(name)
+        if found:
+            return found
+    raise ExportError(
+        "no Chrome-family browser found. Install Google Chrome, Chromium "
+        f"or Microsoft Edge, or set {BROWSER_ENV} to its executable."
+    )
+
+
+def _require_posix() -> None:
+    """Raise if video export cannot run on this operating system."""
+    if sys.platform == "win32":
+        raise ExportError("video export needs macOS or Linux")
+
+
+def _spawn_browser(
+    executable: str,
+    args: list[str],
+    to_browser_r: int,
+    from_browser_w: int,
+) -> int:
+    """Start ``executable`` with the DevTools pipe on fds 3 and 4.
+
+    ``to_browser_r`` becomes fd 3 and ``from_browser_w`` becomes fd 4.
+    Returns the child pid. macOS and Linux only. The platform test is
+    inline so the type checker can see ``os.posix_spawn`` on Unix.
+    """
+    if sys.platform == "win32":
+        raise ExportError("video export needs macOS or Linux")
+    actions = [
+        (os.POSIX_SPAWN_DUP2, to_browser_r, 3),
+        (os.POSIX_SPAWN_DUP2, from_browser_w, 4),
+        # Browser chatter is not the user's business.
+        (os.POSIX_SPAWN_OPEN, 1, os.devnull, os.O_WRONLY, 0),
+        (os.POSIX_SPAWN_OPEN, 2, os.devnull, os.O_WRONLY, 0),
+    ]
+    return os.posix_spawn(executable, args, os.environ, file_actions=actions)
+
+
+class Browser:
+    """A headless browser with one page, driven over the DevTools pipe."""
+
+    def __init__(self, executable: str, width: int, height: int) -> None:
+        _require_posix()
+        self._profile = tempfile.mkdtemp(prefix="concurim-browser-")
+        self._next_id = 0
+        self._buffer = b""
+        to_browser_r, self._to_browser = os.pipe()
+        self._from_browser, from_browser_w = os.pipe()
+        args = [
+            executable,
+            "--headless=new",
+            "--remote-debugging-pipe",
+            f"--user-data-dir={self._profile}",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--hide-scrollbars",
+            "--force-device-scale-factor=1",
+            "--font-render-hinting=none",
+            "--disable-background-timer-throttling",
+            "--mute-audio",
+            f"--window-size={width},{height}",
+            "about:blank",
+        ]
+        self._pid = _spawn_browser(
+            executable, args, to_browser_r, from_browser_w
+        )
+        os.close(to_browser_r)
+        os.close(from_browser_w)
+        self._session = self._open_page(width, height)
+
+    def _open_page(self, width: int, height: int) -> str:
+        target = self.call("Target.createTarget", {"url": "about:blank"})
+        attached = self.call(
+            "Target.attachToTarget",
+            {"targetId": target["targetId"], "flatten": True},
+        )
+        session = attached["sessionId"]
+        self.call(
+            "Emulation.setDeviceMetricsOverride",
+            {
+                "width": width,
+                "height": height,
+                "deviceScaleFactor": 1,
+                "mobile": False,
+            },
+            session,
+        )
+        self.call("Page.enable", session=session)
+        return session
+
+    def call(
+        self,
+        method: str,
+        params: dict[str, Any] | None = None,
+        session: str | None = None,
+    ) -> dict[str, Any]:
+        """Send one command and return its result (events are skipped)."""
+        self._next_id += 1
+        message: dict[str, Any] = {
+            "id": self._next_id,
+            "method": method,
+            "params": params or {},
+        }
+        if session:
+            message["sessionId"] = session
+        os.write(self._to_browser, json.dumps(message).encode() + b"\0")
+        while True:
+            reply = self._read()
+            if reply.get("id") != self._next_id:
+                continue
+            if "error" in reply:
+                raise ExportError(f"{method}: {reply['error']}")
+            return reply.get("result", {})
+
+    def _read(self) -> dict[str, Any]:
+        while b"\0" not in self._buffer:
+            ready, _, _ = select.select(
+                [self._from_browser], [], [], _TIMEOUT_S
+            )
+            chunk = os.read(self._from_browser, 1 << 16) if ready else b""
+            if not chunk:
+                raise ExportError("the browser stopped responding")
+            self._buffer += chunk
+        raw, self._buffer = self._buffer.split(b"\0", 1)
+        return json.loads(raw)
+
+    def page(
+        self, method: str, params: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        """Send a command to the page."""
+        return self.call(method, params, self._session)
+
+    def evaluate(self, expression: str) -> Any:
+        """Run JavaScript in the page (awaiting promises); return the value."""
+        out = self.page(
+            "Runtime.evaluate",
+            {
+                "expression": expression,
+                "awaitPromise": True,
+                "returnByValue": True,
+            },
+        )
+        if "exceptionDetails" in out:
+            raise ExportError(f"page error: {out['exceptionDetails']}")
+        return out["result"].get("value")
+
+    def screenshot(self) -> bytes:
+        """The visible page as PNG bytes."""
+        out = self.page("Page.captureScreenshot", {"format": "png"})
+        return base64.b64decode(out["data"])
+
+    def close(self) -> None:
+        """Stop the browser and delete its temporary profile."""
+        for fd in (self._to_browser, self._from_browser):
+            with contextlib.suppress(OSError):
+                os.close(fd)
+        try:
+            os.kill(self._pid, signal.SIGTERM)
+            os.waitpid(self._pid, 0)
+        except (ProcessLookupError, ChildProcessError):
+            pass
+        shutil.rmtree(self._profile, ignore_errors=True)