process-bigraph 1.8.4__tar.gz → 1.8.5__tar.gz

{process_bigraph-1.8.4/process_bigraph.egg-info → process_bigraph-1.8.5}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: process-bigraph
-Version: 1.8.4
+Version: 1.8.5
 Summary: protocol and execution for compositional systems biology
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.md
 Requires-Dist: bigraph-schema>=1.6.0
+Requires-Dist: requests>=2.31
 Requires-Dist: matplotlib
 Requires-Dist: pint>=0.24.4
 Requires-Dist: scipy>=1.8

{process_bigraph-1.8.4 → process_bigraph-1.8.5}/process_bigraph/composite.py

@@ -18,7 +18,6 @@ import math
 import inspect
 import time as _time
 import contextvars
-import numpy as np
 
 
 # Set by ``Composite.run`` for the duration of the run loop so Steps
@@ -38,76 +37,14 @@ def get_current_composite():
 
 
 # ---------------------------------------------------------------------------
-# Optional invocation tracing
+# Observability
 # ---------------------------------------------------------------------------
-# When the env var ``PROCESS_BIGRAPH_TRACE_FILE`` is set to a writable path,
-# every invocation of ``Composite.process_update`` (i.e. every Process/Step
-# call routed through the framework) writes one JSONL record describing the
-# call: path, class, global_time, interval, an input summary, and an output
-# summary. Useful for diagnosing per-step divergence between two runs by
-# diffing two trace files. No overhead when the env var is unset.
-_TRACE_PATH = os.environ.get('PROCESS_BIGRAPH_TRACE_FILE')
-_TRACE_FH = open(_TRACE_PATH, 'a', buffering=1) if _TRACE_PATH else None
-
-# Default for Composite._profile_per_process (per-path invoke attribution in
-# TimingSummary.per_process). Off by default — the write is on the hot path —
-# and flippable per-run via the env var without a code change.
-_PROFILE_PER_PROCESS_DEFAULT = bool(
-    os.environ.get('PROCESS_BIGRAPH_PROFILE_PROCESSES'))
-
-
-def _summarize_value(value, depth=0):
-    """Lightweight, JSON-safe summary of an update fragment.
-
-    Tries to surface the values most useful for diffing two traces:
-    - Scalars and short strings inlined.
-    - Numpy arrays summarized as ``{shape, sum, mean, head}``.
-    - Dicts recursed (capped depth).
-    - Lists/tuples shown as their first few items.
-    """
-    if depth > 3:
-        return f'<{type(value).__name__}>'
-    if value is None or isinstance(value, (bool, int, float, str)):
-        return value
-    if isinstance(value, np.ndarray):
-        try:
-            return {
-                '_np': True,
-                'shape': list(value.shape),
-                'dtype': str(value.dtype),
-                'sum': float(value.sum()) if value.dtype.kind in 'fi' else None,
-                'head': value.flatten()[:5].tolist() if value.size else [],
-            }
-        except Exception:
-            return f'<ndarray shape={value.shape}>'
-    if isinstance(value, dict):
-        return {k: _summarize_value(v, depth + 1) for k, v in list(value.items())[:32]}
-    if isinstance(value, (list, tuple)):
-        return [_summarize_value(v, depth + 1) for v in value[:10]]
-    return f'<{type(value).__name__}>'
-
-
-def _trace_invoke(path, instance, state, interval, update):
-    """Append one JSONL record describing a process/step invocation."""
-    if _TRACE_FH is None:
-        return
-    try:
-        gt = state.get('global_time') if isinstance(state, dict) else None
-        rec = {
-            'path': list(path) if isinstance(path, tuple) else path,
-            'cls': type(instance).__name__,
-            'gt': gt,
-            'interval': interval,
-            'input': _summarize_value(state),
-            'output': _summarize_value(update),
-        }
-        _TRACE_FH.write(json.dumps(rec, default=str) + '\n')
-    except Exception as exc:
-        # Tracing must never break a sim. Log a single line and continue.
-        try:
-            _TRACE_FH.write(json.dumps({'trace_error': str(exc)}) + '\n')
-        except Exception:
-            pass
+# Invocation tracing, per-process profiling, the run heartbeat and exception
+# context all live in ``process_bigraph.events`` (one module, one set of
+# switches -- see its docstring). ``_summarize_value`` is re-exported here for
+# callers that imported it from ``composite``.
+from process_bigraph import events as _events
+from process_bigraph.events import _summarize_value  # noqa: F401  (re-export)
 
 from dataclasses import dataclass, field
 from typing import (
@@ -1302,6 +1239,16 @@ class Composite(Process):
     # Initialization & Configuration
     # ==============================
 
+    @property
+    def _em(self) -> '_events.EventEmitter':
+        """The process-wide event emitter (process_bigraph.events); resolved
+        lazily so a Composite built before ``events.configure()`` still
+        picks up the configured sinks."""
+        em = self.__dict__.get('_events')
+        if em is None:
+            em = self._events = _events.get_emitter()
+        return em
+
     def initialize(self, config: Optional[Dict[str, Any]] = None) -> None:
         """
         Initialize the composite model from its config.
@@ -1505,7 +1452,19 @@ class Composite(Process):
         # ``run()`` to get the per-path breakdown. Defaults to the
         # ``PROCESS_BIGRAPH_PROFILE_PROCESSES`` env var so a run can be profiled
         # without code changes.
-        self._profile_per_process: bool = _PROFILE_PER_PROCESS_DEFAULT
+        self._events = _events.get_emitter()
+        self._profile_per_process: bool = self._events.detail_timing
+        # ``run.start``/``run.end`` rate limit (per Composite instance). A
+        # protocol runtime drives one Composite.run per remote step, so an
+        # unthrottled pair is 2 x cells x ticks events. The first run and any
+        # run that ends in error always emit; otherwise at most one pair per
+        # ``heartbeat_s``, and the runs skipped in between are folded into the
+        # next ``run.end`` (``runs=N`` with the summed totals).
+        self._run_emitted_at: Optional[float] = None
+        self._runs_folded: int = 0
+        self._folded_total: float = 0.0
+        self._folded_process_time: float = 0.0
+        self._folded_ticks: int = 0
 
         # Precompile view/project operations for fast runtime access.
         self._compiled_links = {}
@@ -1548,7 +1507,13 @@ class Composite(Process):
         for rt in runtimes:
             flush = getattr(rt, 'flush_pending', None)
             if flush is not None:
-                flush()
+                try:
+                    flush()
+                except BaseException as exc:
+                    self._em.exception(
+                        exc, event_name='runtime.error', runtime=type(rt).__name__,
+                        global_time=self.state.get('global_time'))
+                    raise
 
     def _partition_processes_by_runtime(self):
         """Inspect every process_path and split into:
@@ -2617,6 +2582,29 @@ class Composite(Process):
         self._per_process_time = {}
         run_start = _time.monotonic()
 
+        # Observability (process_bigraph.events): a ``run`` span when spans
+        # are on, ``run.start``/``run.end`` always (no-ops when no sink is
+        # configured), heartbeat counters reset per run.
+        em = self._events = _events.get_emitter()
+        if em.detail_timing:
+            self._profile_per_process = True
+        self._run_ticks = 0     # per-Composite, so nested runs do not clobber the outer count
+        # Rate limit: emit this pair if it is the first run on this Composite or
+        # ``heartbeat_s`` has elapsed since the last emitted pair. An error at
+        # the end overrides the decision (see ``finally``), because a failure is
+        # never the event to drop.
+        emit_pair = (
+            self._run_emitted_at is None
+            or (run_start - self._run_emitted_at) >= em.heartbeat_s)
+        span = em.start_span('run', interval=interval) if (
+            em.enabled and em.detail_spans and emit_pair) else None
+        if emit_pair:
+            em.event('run.start', interval=interval,
+                     n_processes=len(self.process_paths), n_steps=len(self.step_paths),
+                     parallel=bool(getattr(self, '_parallel_processes', False)),
+                     global_time=self.state.get('global_time'))
+        status, error = 'ok', None
+
         # Set the current-Composite contextvar so nested Steps can
         # reach the parent Composite (e.g. for CompositeDivision to
         # read parent cell tree state without going through wires).
@@ -2625,8 +2613,51 @@ class Composite(Process):
         _cc_token = current_composite_var.set(self)
         try:
             return self._run_inner(interval, force_complete, run_start)
+        except BaseException as exc:
+            status, error = 'error', f'{type(exc).__name__}: {exc}'
+            raise
         finally:
             current_composite_var.reset(_cc_token)
+            if em.enabled:
+                total = _time.monotonic() - run_start
+                # An error always reports, even when the rate limit had
+                # suppressed this run's ``run.start``.
+                report = emit_pair or status == 'error'
+                if not report:
+                    self._runs_folded += 1
+                    self._folded_total += total
+                    self._folded_process_time += self.process_update_time
+                    self._folded_ticks += self._run_ticks
+                else:
+                    runs = self._runs_folded + 1
+                    agg_total = self._folded_total + total
+                    agg_process = self._folded_process_time + self.process_update_time
+                    payload = {
+                        'status': status, 'error': error,
+                        'runs': runs,
+                        'total': round(agg_total, 6),
+                        'process_time': round(agg_process, 6),
+                        'framework_time': round(agg_total - agg_process, 6),
+                        'ticks': self._folded_ticks + self._run_ticks,
+                        'global_time': self.state.get('global_time'),
+                    }
+                    if not emit_pair:
+                        # The pair is unbalanced: this end has no matching
+                        # start, because the error overrode the rate limit.
+                        payload['start_suppressed'] = True
+                    if self._per_process_time:
+                        top = sorted(self._per_process_time.items(), key=lambda kv: kv[1], reverse=True)[:5]
+                        payload['top5'] = [
+                            ['/'.join(str(p) for p in k) if isinstance(k, tuple) else str(k), round(v, 6)]
+                            for k, v in top]
+                    em.event('run.end', level='error' if status == 'error' else 'info', **payload)
+                    self._run_emitted_at = _time.monotonic()
+                    self._runs_folded = 0
+                    self._folded_total = 0.0
+                    self._folded_process_time = 0.0
+                    self._folded_ticks = 0
+                if span is not None:
+                    span.end(status, error)
 
     def _run_inner(self, interval: float, force_complete: bool,
                    run_start: float) -> None:
@@ -2637,8 +2668,11 @@ class Composite(Process):
             self.run_steps(self.to_run)
             self.to_run = []
 
+        em = self._em
         while self.state['global_time'] < end_time or force_complete:
             full_step = math.inf
+            self._run_ticks = getattr(self, '_run_ticks', 0) + 1
+            em.heartbeat(global_time=self.state['global_time'])
 
             # Partition processes: ones whose runtime opts into batched
             # tick_lifecycle vs the regular per-process path. The
@@ -2991,7 +3025,11 @@ class Composite(Process):
 
         if len(due) > 1:
             pool = self._get_step_executor(len(due))
-            results = list(pool.map(_invoke, due))
+            # Carry the observability span context (a ContextVar) into the
+            # worker threads: each call runs in its own copy of this thread's
+            # context, so events emitted from a worker carry the run's ids.
+            snapshot = contextvars.copy_context()
+            results = list(pool.map(lambda item: snapshot.copy().run(_invoke, item), due))
         else:
             results = [_invoke(due[0])]
 
@@ -3034,9 +3072,25 @@ class Composite(Process):
         # Strip schema-specific metadata from the state (once, unless caller did)
         clean_state = states if already_clean else strip_schema_keys(states)
 
-        # Invoke the process and retrieve a wrapped SyncUpdate object
+        # Invoke the process and retrieve a wrapped SyncUpdate object.
+        # On failure, record WHICH process/step, at what global_time, with a
+        # summary of the state it was handed (process_bigraph.events), then
+        # re-raise the ORIGINAL exception unchanged -- callers may classify
+        # control-flow exceptions by type, so the type must propagate as is.
         t0 = _time.monotonic()
-        update = process['instance'].invoke(clean_state, interval)
+        try:
+            update = process['instance'].invoke(clean_state, interval)
+        except BaseException as exc:
+            self._em.exception(
+                exc,
+                path='/'.join(str(p) for p in path) if isinstance(path, (list, tuple)) else str(path),
+                cls=type(process['instance']).__name__,
+                address=process.get('address'),
+                is_step=(interval == -1.0),
+                interval=interval,
+                global_time=self.state.get('global_time') if isinstance(self.state, dict) else None,
+                state_summary=_events.summarize_state(clean_state))
+            raise
         dt = _time.monotonic() - t0
         self.process_update_time += dt
         # Per-process attribution is opt-in (a dict write on the hot path);
@@ -3046,7 +3100,7 @@ class Composite(Process):
         if self._profile_per_process:
             key = tuple(path) if isinstance(path, (list, tuple)) else (path,)
             self._per_process_time[key] = self._per_process_time.get(key, 0.0) + dt
-        if _TRACE_FH is not None:
+        if self._em.detail_invoke:
             # Resolve SyncUpdate / Defer to plain dict for the trace; the
             # invoke return is opaque (SyncUpdate wraps a dict). We only need
             # a snapshot for the trace, so pull .get() if available.
@@ -3054,7 +3108,7 @@ class Composite(Process):
                 resolved = update.get() if hasattr(update, 'get') else update
             except Exception:
                 resolved = '<unresolvable>'
-            _trace_invoke(path, process['instance'], clean_state, interval, resolved)
+            self._em.invoke(path, process['instance'], clean_state, interval, resolved)
 
         # Return a deferred object that will project the update when requested.
         # The projector is the module-level ``_project_process_update`` (not a
@@ -3219,8 +3273,14 @@ class Composite(Process):
             finally:
                 uninstall_reconcile_sink(prev_sink)
             update_paths = summary.paths
+            self._em.count(leaf_paths_touched=len(update_paths))
             if summary.has_structural:
                 had_structural_sentinels = True
+                self._em.count(structural_changes=1)
+                self._em.event(
+                    'structure.changed', n_paths=len(update_paths),
+                    sample_paths=['/'.join(str(p) for p in path) for path in update_paths[:8]],
+                    global_time=self.state.get('global_time'))
 
             if combined_update:
                 # NOTE: previously this path also forced

{process_bigraph-1.8.4 → process_bigraph-1.8.5}/process_bigraph/composite_generator.py

@@ -42,6 +42,11 @@ class GeneratorEntry:
     # a Study is built on top of this composite the dashboard merges these
     # defaults into its visualizations list; Studies can still declare extras.
     visualizations: list[dict] = field(default_factory=list)
+    # Canonical analyses that ship with this composite. Each entry is a
+    # Study-spec analysis dict ({name, params?}) or a scale-grouped block,
+    # matching CompositeSpec.analyses. Merged under a run's config-declared
+    # analyses when a composite runs standalone (config wins).
+    analyses: list[dict] = field(default_factory=list)
     # Emitter(s) this composite ships as its default observation sink. Each
     # entry is a lightweight ``{address, config, paths?}`` dict: ``address`` is
     # the registered emitter link (e.g. ``"local:ParquetEmitter"``), ``config``
@@ -80,6 +85,7 @@ def _entry_for(spec) -> GeneratorEntry:
         module=spec.module,
         default_n_steps=spec.default_n_steps,
         visualizations=spec.visualizations,
+        analyses=spec.analyses,
         emitters=spec.emitters,
         core_extensions=spec.core_extensions,
     )
@@ -155,6 +161,7 @@ class _RegistryView:
                 module=getattr(entry, "module", ""),
                 default_n_steps=getattr(entry, "default_n_steps", None),
                 visualizations=list(getattr(entry, "visualizations", []) or []),
+                analyses=list(getattr(entry, "analyses", []) or []),
                 emitters=list(getattr(entry, "emitters", []) or []),
                 core_extensions=list(getattr(entry, "core_extensions", []) or []),
             )
@@ -180,6 +187,7 @@ def composite_generator(
     description: str = "",
     parameters: dict[str, dict] | None = None,
     visualizations: list[dict] | None = None,
+    analyses: list[dict] | None = None,
     emitters: list[dict] | None = None,
     default_n_steps: int | None = None,
     core_extensions: list[Callable[[Any], Any]] | None = None,
@@ -209,6 +217,12 @@ def composite_generator(
     composite author considers canonical) without having to hand-author them
     in every Study spec.
 
+    `analyses` declares the canonical analyses that ship with this composite.
+    Each entry is a Study-spec analysis dict (``{name, params?}``) or a
+    scale-grouped block, matching ``CompositeSpec.analyses``. Mirrors
+    ``visualizations``: a run built on this composite standalone merges these
+    defaults under any config-declared analyses (config wins).
+
     `emitters` (optional) declares the default observation sink(s) this
     composite ships with. Each entry is a lightweight
     ``{"address": "local:ParquetEmitter", "config": {...}, "paths": [...]}``
@@ -266,6 +280,7 @@ def composite_generator(
             description=description,
             parameters=parameters,
             visualizations=visualizations,
+            analyses=analyses,
             emitters=validated_emitters,
             default_n_steps=default_n_steps,
             core_extensions=core_extensions,

{process_bigraph-1.8.4 → process_bigraph-1.8.5}/process_bigraph/emitter.py

@@ -584,6 +584,16 @@ class ConsoleEmitter(Emitter):
 def tree_copy(state):
     '''Deep copy utility for nested simulation state (excluding Edge instances).'''
     if isinstance(state, dict):
+        # A realized process/step node carries its live Edge under 'instance'.
+        # Its remaining keys (address/config/inputs/outputs/interval/_type) are
+        # static wiring, not observable state — deep-copying them every tick is
+        # a large, pointless cost for composites that embed a process under each
+        # agent (e.g. cells/<id>/chemotaxis: ~34 of 49 nodes/cell are the process
+        # spec, a ~15x blow-up paid every emit). The live Edge was already
+        # dropped below; drop the whole node so process specs never reach the
+        # emitted history.
+        if isinstance(state.get('instance'), Edge):
+            return None
         return {k: v for k, v in ((k, tree_copy(v)) for k, v in state.items()) if v is not None}
     if isinstance(state, np.ndarray):
         return state.copy()