onec-interactive-runtime-core 0.1.19__py3-none-any.whl

onec_runtime/performance_profile.py

@@ -0,0 +1,245 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from time import perf_counter_ns, process_time_ns
+from typing import Callable, TypeVar
+
+
+T = TypeVar("T")
+
+PARSER_CALL_COUNTERS = (
+    "full_module_parses",
+    "delta_method_parses",
+    "packaging_validation_parses",
+)
+_INTERNAL_PARSER_CALL_COUNTERS = (*PARSER_CALL_COUNTERS, "worker_profile_parses")
+
+_PING_PARSE_PHASES = frozenset(
+    {
+        "rdbg.ping.parse_xml",
+        "rdbg.ping.parse_targets",
+        "rdbg.ping.parse_stops",
+        "rdbg.ping.parse_local_variables",
+        "rdbg.ping.parse_evaluations",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class PhaseEvent:
+    sequence: int
+    phase: str
+    wall_ns: int
+    cpu_ns: int
+    input_bytes: int = 0
+    output_bytes: int = 0
+    item_count: int = 0
+    page_start: int | None = None
+    result_id: str = ""
+    error_present: bool = False
+
+    def as_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class ParserCallCounters:
+    full_module_parses: int = 0
+    delta_method_parses: int = 0
+    packaging_validation_parses: int = 0
+    worker_profile_parses: int = 0
+
+    def as_tuple(self) -> tuple[int, int, int]:
+        return (
+            self.full_module_parses,
+            self.delta_method_parses,
+            self.packaging_validation_parses,
+        )
+
+    def as_dict(self) -> dict[str, int]:
+        return {
+            "full_module_parses": self.full_module_parses,
+            "delta_method_parses": self.delta_method_parses,
+            "packaging_validation_parses": self.packaging_validation_parses,
+        }
+
+
+class PhaseRecorder:
+    """Records aggregate timings without retaining runtime values or payloads."""
+
+    def __init__(
+        self,
+        *,
+        wall_clock_ns: Callable[[], int] = perf_counter_ns,
+        cpu_clock_ns: Callable[[], int] = process_time_ns,
+    ) -> None:
+        self.events: list[PhaseEvent] = []
+        self._wall_clock_ns = wall_clock_ns
+        self._cpu_clock_ns = cpu_clock_ns
+        self._parser_calls = {name: 0 for name in _INTERNAL_PARSER_CALL_COUNTERS}
+
+    @property
+    def parser_calls(self) -> ParserCallCounters:
+        return ParserCallCounters(**self._parser_calls)
+
+    def measure_parser(
+        self,
+        counter: str,
+        operation: Callable[[], T],
+    ) -> T:
+        self.record_parser_call(counter)
+        return self.measure("semantic_parse", operation)
+
+    def record_parser_call(self, counter: str) -> None:
+        if counter not in self._parser_calls:
+            raise ValueError("parser call counter is invalid")
+        self._parser_calls[counter] += 1
+
+    def record_duration(
+        self,
+        phase: str,
+        *,
+        wall_ns: int,
+        cpu_ns: int = 0,
+        input_bytes: int = 0,
+        output_bytes: int = 0,
+        item_count: int = 0,
+        result_id: str = "",
+    ) -> None:
+        if not isinstance(phase, str) or not phase:
+            raise ValueError("phase must not be empty")
+        counters = (wall_ns, cpu_ns, input_bytes, output_bytes, item_count)
+        if any(type(value) is not int or value < 0 for value in counters):
+            raise ValueError("profile counters must be non-negative integers")
+        self.events.append(
+            PhaseEvent(
+                sequence=len(self.events) + 1,
+                phase=phase,
+                wall_ns=wall_ns,
+                cpu_ns=cpu_ns,
+                input_bytes=input_bytes,
+                output_bytes=output_bytes,
+                item_count=item_count,
+                result_id=result_id,
+            )
+        )
+
+    def measure(
+        self,
+        phase: str,
+        operation: Callable[[], T],
+        *,
+        input_bytes: int = 0,
+        page_start: int | None = None,
+        result_id: str = "",
+        output_bytes: Callable[[T], int] | None = None,
+        item_count: Callable[[T], int] | None = None,
+    ) -> T:
+        if not phase:
+            raise ValueError("phase must not be empty")
+        if input_bytes < 0 or (page_start is not None and page_start < 0):
+            raise ValueError("profile counters must be non-negative")
+        wall_started = self._wall_clock_ns()
+        cpu_started = self._cpu_clock_ns()
+        try:
+            result = operation()
+        except BaseException:
+            self._append(
+                phase,
+                wall_started,
+                cpu_started,
+                input_bytes=input_bytes,
+                page_start=page_start,
+                result_id=result_id,
+                error_present=True,
+            )
+            raise
+        measured_output = output_bytes(result) if output_bytes is not None else 0
+        measured_items = item_count(result) if item_count is not None else 0
+        if measured_output < 0 or measured_items < 0:
+            raise ValueError("profile counters must be non-negative")
+        self._append(
+            phase,
+            wall_started,
+            cpu_started,
+            input_bytes=input_bytes,
+            output_bytes=measured_output,
+            item_count=measured_items,
+            page_start=page_start,
+            result_id=result_id,
+        )
+        return result
+
+    def _append(
+        self,
+        phase: str,
+        wall_started: int,
+        cpu_started: int,
+        *,
+        input_bytes: int = 0,
+        output_bytes: int = 0,
+        item_count: int = 0,
+        page_start: int | None = None,
+        result_id: str = "",
+        error_present: bool = False,
+    ) -> None:
+        self.events.append(
+            PhaseEvent(
+                sequence=len(self.events) + 1,
+                phase=phase,
+                wall_ns=self._wall_clock_ns() - wall_started,
+                cpu_ns=self._cpu_clock_ns() - cpu_started,
+                input_bytes=input_bytes,
+                output_bytes=output_bytes,
+                item_count=item_count,
+                page_start=page_start,
+                result_id=result_id,
+                error_present=error_present,
+            )
+        )
+
+
+def summarize_phases(events: list[PhaseEvent]) -> dict[str, object]:
+    by_phase: dict[str, dict[str, int]] = {}
+    for event in events:
+        totals = by_phase.setdefault(
+            event.phase,
+            {
+                "calls": 0,
+                "wall_ns": 0,
+                "cpu_ns": 0,
+                "input_bytes": 0,
+                "output_bytes": 0,
+                "item_count": 0,
+                "error_count": 0,
+            },
+        )
+        totals["calls"] += 1
+        totals["wall_ns"] += event.wall_ns
+        totals["cpu_ns"] += event.cpu_ns
+        totals["input_bytes"] += event.input_bytes
+        totals["output_bytes"] += event.output_bytes
+        totals["item_count"] += event.item_count
+        totals["error_count"] += int(event.error_present)
+    ping_response_bytes = by_phase.get("rdbg.ping.request", {}).get(
+        "output_bytes", 0
+    )
+    ping_parser_input_bytes = sum(
+        by_phase.get(phase, {}).get("input_bytes", 0)
+        for phase in _PING_PARSE_PHASES
+    )
+    return {
+        "event_count": len(events),
+        "by_phase": by_phase,
+        "derived": {
+            "ping_response_bytes": ping_response_bytes,
+            "ping_parser_input_bytes": ping_parser_input_bytes,
+            "ping_xml_parse_amplification": (
+                ping_parser_input_bytes / ping_response_bytes
+                if ping_response_bytes
+                else 0.0
+            ),
+            "profiled_wall_ns": sum(event.wall_ns for event in events),
+            "profiled_cpu_ns": sum(event.cpu_ns for event in events),
+        },
+    }

onec_runtime/privacy.py

@@ -0,0 +1,263 @@
+from __future__ import annotations
+
+from dataclasses import fields, is_dataclass
+from hashlib import sha256
+import re
+from typing import Any
+
+from onec_runtime.runtime_contracts import sanitize_normalized_diagnostic
+from onec_runtime.bsl import NormalizedDiagnostic
+
+
+_PLATFORM_DIAGNOSTIC_LIMIT = 4_096
+_SENSITIVE_DIAGNOSTIC_KEY = (
+    r"(?:"
+    r"[0-9A-Za-z_.-]*"
+    r"(?:password|passwd|passphrase|secret|token|authorization|credential(?:s)?)"
+    r"(?:[_.-]?(?:id|key|value|header))?|"
+    r"rdbg[_\-.]?(?:session|connection|client|server|process|subject|"
+    r"target|object|property|seance)"
+    r"(?:[_\-.]?(?:pid|id|key))?|"
+    r"rdbg[0-9A-Za-z_.-]*(?:pid|id|key)|"
+    r"(?:process|session|connection|client|server|subject|runtime|worker)"
+    r"[_\-.]?(?:pid|id|token|key|secret|credential(?:s)?)|"
+    r"(?:process|session|connection|client|server|subject|runtime|worker)"
+    r"[ \t]+(?:pid|id|token|key|secret|credential(?:s)?)|"
+    r"(?:access|refresh|auth|api)[_\-.]?(?:key|secret|credential|header)|"
+    r"pid"
+    r")"
+)
+_SENSITIVE_ASSIGNMENT_RE = re.compile(
+    rf"""
+    (?P<head>
+        (?:
+            ["']{_SENSITIVE_DIAGNOSTIC_KEY}["']
+            |
+            (?<![0-9A-Za-z_]){_SENSITIVE_DIAGNOSTIC_KEY}(?![0-9A-Za-z_])
+        )
+        (?:[ \t]*(?:=|:)[ \t]*|[ \t]+)
+    )
+    (?P<value>
+        "(?:\\.|[^"\\])*"
+        |'(?:\\.|[^'\\])*'
+        |[^\r\n,;}}\]]+
+    )
+    """,
+    re.IGNORECASE | re.VERBOSE,
+)
+_AUTHORIZATION_RE = re.compile(
+    r"\b(?:bearer|basic)\s+[^\s,;}}\]]+",
+    re.IGNORECASE,
+)
+
+
+def diagnostic_to_public_wire(value: NormalizedDiagnostic) -> dict[str, object]:
+    """Render one canonical visible-only diagnostic or fail closed."""
+    try:
+        safe = sanitize_normalized_diagnostic(value)
+        if safe is None:
+            return {}
+        visible_location = safe.visible_location
+        related = safe.related_visible_span
+        return {
+            "diagnostic_id": safe.diagnostic_id,
+            "runtime_summary": safe.runtime_summary,
+            "stage": safe.stage.value,
+            "mapping_confidence": safe.mapping_confidence.value,
+            "visible_location": (
+                None
+                if visible_location is None
+                else {
+                    "line": visible_location.line,
+                    "column": visible_location.column,
+                    "span": _span_wire(visible_location.span),
+                }
+            ),
+            "related_visible_span": (
+                None if related is None else _span_wire(related)
+            ),
+            "excerpt": None,
+            "synthetic_region": safe.synthetic_region,
+        }
+    except BaseException:
+        return {}
+
+
+def diagnostic_to_expert_wire(value: NormalizedDiagnostic) -> dict[str, object]:
+    """Render the bounded expert allowlist without source or runtime identity."""
+    try:
+        safe = sanitize_normalized_diagnostic(value)
+        public = diagnostic_to_public_wire(value)
+        if safe is None or not public:
+            return {}
+        lowered = safe.lowered_location
+        platform, platform_truncated, platform_redacted = (
+            bounded_platform_diagnostic(
+                safe.platform_diagnostic,
+                truncated=safe.platform_diagnostic_truncated,
+                redacted=safe.platform_diagnostic_redacted,
+            )
+        )
+        return {
+            **public,
+            "lowered_location": (
+                None
+                if lowered is None
+                else {
+                    "line": lowered.line,
+                    "column": lowered.column,
+                    "offset": lowered.offset,
+                    "span": _span_wire(lowered.span),
+                }
+            ),
+            "platform_diagnostic": platform,
+            "platform_diagnostic_sha256": safe.platform_diagnostic_sha256,
+            "platform_diagnostic_truncated": platform_truncated,
+            "platform_diagnostic_redacted": platform_redacted,
+            "execution_artifact_sha256": safe.execution_artifact_sha256,
+            "source_map_sha256": safe.source_map_sha256,
+            # RuntimeApi can bind these exact nullable fields from its
+            # pre-dispatch provenance; the diagnostic itself never invents it.
+            "worker_generation": None,
+            "worker_manifest_sha256": None,
+        }
+    except BaseException:
+        return {}
+
+
+def bounded_platform_diagnostic(
+    value: str | None,
+    *,
+    truncated: bool,
+    redacted: bool,
+) -> tuple[str | None, bool, bool]:
+    """Bound and redact credential/process identities in expert-only prose."""
+    if type(truncated) is not bool or type(redacted) is not bool:
+        return None, True, True
+    if value is None:
+        return None, truncated, redacted
+    if type(value) is not str:
+        return None, True, True
+    bounded = value[:_PLATFORM_DIAGNOSTIC_LIMIT]
+    input_truncated = len(value) > _PLATFORM_DIAGNOSTIC_LIMIT
+    bounded, assignment_count = _SENSITIVE_ASSIGNMENT_RE.subn(
+        lambda match: f"{match.group('head')}<redacted>",
+        bounded,
+    )
+    bounded, authorization_count = _AUTHORIZATION_RE.subn(
+        "<redacted credential>",
+        bounded,
+    )
+    expansion_truncated = len(bounded) > _PLATFORM_DIAGNOSTIC_LIMIT
+    if len(bounded) > _PLATFORM_DIAGNOSTIC_LIMIT:
+        bounded = bounded[:_PLATFORM_DIAGNOSTIC_LIMIT]
+    return (
+        bounded,
+        truncated or input_truncated or expansion_truncated,
+        redacted or assignment_count > 0 or authorization_count > 0,
+    )
+
+
+def _span_wire(value: object) -> dict[str, object]:
+    return {
+        "start": getattr(value, "start"),
+        "end": getattr(value, "end"),
+    }
+
+
+def public_artifact_value(value: Any) -> Any:
+    """Recursively redact reversible BSL sources from automatic evidence paths."""
+    kind = type(value).__name__
+    if kind == "ModuleLocation":
+        return {
+            "module_type": value.module_type,
+            "line": value.line,
+            "extension_name": value.extension_name,
+        }
+    if kind == "WorkerGenerationDebugView":
+        return {
+            "handle": public_artifact_value(value.handle),
+            "module_count": len(value.modules),
+        }
+    if kind == "WorkerModuleDebugView":
+        return {
+            "source_unit": public_artifact_value(value.source_unit),
+            "canonical_module": value.canonical_module,
+            "artifact_sha256": value.artifact_sha256,
+            "source_map_sha256": value.source_map_sha256,
+        }
+    if kind == "WorkerBreakpointPlan":
+        return {
+            "proposal_id": value.proposal_id,
+            "expected_catalog_version": value.expected_catalog_version,
+            "result_id": value.result_id,
+            "desired_slot_count": len(value.desired_slots),
+            "removed_ids": public_artifact_value(value.removed_ids),
+            "next_snapshot": public_artifact_value(value.next_snapshot),
+        }
+    if kind == "WorkerBreakpointBinding":
+        return {
+            "breakpoint_id": value.breakpoint_id,
+            "generation": public_artifact_value(value.generation),
+            "location": public_artifact_value(value.location),
+        }
+    if kind == "WorkerFrameProvenance":
+        return "<redacted worker frame provenance>"
+    if kind == "WorkerArtifact":
+        return {
+            "logical_name": value.logical_name,
+            "source_sha256": value.source_sha256,
+            "source_map_sha256": value.source_map_sha256,
+            "artifact_sha256": value.artifact_sha256,
+            "source_provenance": public_artifact_value(value.source_provenance),
+            "exports": public_artifact_value(value.exports),
+        }
+    if kind == "OperationHandle":
+        return {
+            "operation_id": value.operation_id,
+            "visible_source_sha256": _source_sha256(value.visible_source),
+            "lowered_source_sha256": _source_sha256(value.lowered_source),
+            "messages_intercepted": value.messages_intercepted,
+        }
+    if kind == "CaptureCellResult":
+        return {
+            "operation_id": value.operation_id,
+            "visible_source_sha256": _source_sha256(value.visible_source),
+            "lowered_source_sha256": _source_sha256(value.lowered_source),
+            "result": public_artifact_value(value.result),
+            "messages": public_artifact_value(value.messages),
+        }
+    if kind == "NotebookCell":
+        return {
+            "worker_source_sha256": _source_sha256(value.worker_source),
+            "statement_source_sha256": _source_sha256(value.statement_source),
+            "exports": public_artifact_value(value.exports),
+        }
+    if is_dataclass(value) and not isinstance(value, type):
+        return {
+            item.name: public_artifact_value(getattr(value, item.name))
+            for item in fields(value)
+        }
+    if isinstance(value, dict):
+        return _public_mapping(value)
+    if isinstance(value, (tuple, list, set, frozenset)):
+        return [public_artifact_value(item) for item in value]
+    return value
+
+
+def _source_sha256(source: str) -> str:
+    return sha256(source.encode("utf-8")).hexdigest()
+
+
+def _public_mapping(value: dict[Any, Any]) -> dict[str, Any]:
+    result = {str(key): public_artifact_value(item) for key, item in value.items()}
+    for field_name in (
+        "visible_source",
+        "lowered_source",
+        "worker_source",
+        "statement_source",
+    ):
+        source = result.pop(field_name, None)
+        if isinstance(source, str):
+            result[f"{field_name}_sha256"] = _source_sha256(source)
+    return result