qwen-sci 0.2.2__py3-none-any.whl

src/agents/research_plan_author/run_logging.py

@@ -0,0 +1,209 @@
+"""Structured, redacted console and JSONL logging for Research Plan Author."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import re
+import sys
+from threading import RLock
+from time import perf_counter
+from collections.abc import Iterator, Mapping
+from typing import Any, TextIO
+
+
+AUTHOR_LOGGING_SCHEMA_VERSION = "research_plan_author_run_log_v1"
+_LEVELS = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}
+_CORE_FIELDS = {"timestamp", "level", "run_id", "stage", "event", "status", "elapsed_ms"}
+_SENSITIVE_KEYS = {
+    "prompt",
+    "raw",
+    "raw_response",
+    "response",
+    "source_text",
+    "fulltext",
+    "full_text",
+    "abstract",
+    "content",
+    "api_key",
+    "apikey",
+    "access_token",
+    "token",
+    "secret",
+    "password",
+    "clinical_data",
+    "patient_data",
+    "personal_data",
+    "dangerous_parameters",
+    "restricted_protocol",
+}
+_SECRET_PATTERN = re.compile(
+    r"(?i)(api[_ -]?key|access[_ -]?token|token|password|secret)\s*[:=]\s*[^\s,;]+"
+)
+
+
+class AuthorRunLoggingError(RuntimeError):
+    """Raised when the Author logger cannot write its configured sink."""
+
+
+def _safe_text(value: object, *, limit: int = 2000) -> str:
+    text = _SECRET_PATTERN.sub(lambda match: f"{match.group(1)}=[REDACTED]", str(value or ""))
+    return text[:limit]
+
+
+def _sanitize(value: object, *, key: object = "") -> object:
+    normalized_key = re.sub(r"[^a-z0-9]+", "_", str(key).casefold()).strip("_")
+    if normalized_key in _SENSITIVE_KEYS or normalized_key.endswith("_prompt") or normalized_key.endswith("_raw_response"):
+        return "[REDACTED]"
+    if isinstance(value, Mapping):
+        return {str(item_key): _sanitize(item_value, key=item_key) for item_key, item_value in list(value.items())[:100]}
+    if isinstance(value, (list, tuple, set)):
+        return [_sanitize(item) for item in list(value)[:100]]
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    return _safe_text(value)
+
+
+class AuthorRunLogger:
+    """Emit structured Author events while redacting prompts, sources, and secrets."""
+
+    def __init__(
+        self,
+        run_id: str,
+        *,
+        jsonl_path: str | Path | None = None,
+        console_stream: TextIO | None = None,
+        console_level: str = "INFO",
+        console_enabled: bool = True,
+    ) -> None:
+        self.run_id = _safe_text(run_id, limit=300).strip()
+        if not self.run_id:
+            raise ValueError("run_id must be non-empty")
+        self.console_level = str(console_level or "INFO").upper()
+        if self.console_level not in _LEVELS:
+            raise ValueError(f"Unsupported log level: {console_level}")
+        self.console_stream = console_stream if console_stream is not None else sys.stderr
+        self.console_enabled = bool(console_enabled)
+        self.jsonl_path = Path(jsonl_path).expanduser().resolve() if jsonl_path is not None else None
+        self._file: TextIO | None = None
+        self._lock = RLock()
+        if self.jsonl_path is not None:
+            try:
+                self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+                self._file = self.jsonl_path.open("a", encoding="utf-8", buffering=1)
+            except OSError as error:
+                raise AuthorRunLoggingError(f"Cannot open log file '{self.jsonl_path}': {error}") from error
+
+    def emit(
+        self,
+        stage: str,
+        event: str,
+        *,
+        level: str = "INFO",
+        status: str = "INFO",
+        elapsed_ms: float | None = None,
+        **fields: object,
+    ) -> dict[str, Any]:
+        normalized_level = str(level or "INFO").upper()
+        if normalized_level not in _LEVELS:
+            raise ValueError(f"Unsupported log level: {level}")
+        normalized_stage = _safe_text(stage, limit=160).strip()
+        normalized_event = _safe_text(event, limit=160).strip()
+        if not normalized_stage or not normalized_event:
+            raise ValueError("stage and event must be non-empty")
+        record: dict[str, Any] = {
+            "schema_version": AUTHOR_LOGGING_SCHEMA_VERSION,
+            "timestamp": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
+            "level": normalized_level,
+            "run_id": self.run_id,
+            "stage": normalized_stage,
+            "event": normalized_event,
+            "status": _safe_text(status, limit=100).strip() or "INFO",
+            "elapsed_ms": round(float(elapsed_ms), 3) if elapsed_ms is not None else None,
+        }
+        for key, value in _sanitize(fields).items():
+            if key not in _CORE_FIELDS and key != "schema_version":
+                record[key] = value
+        line = json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+        with self._lock:
+            try:
+                if self._file is not None:
+                    self._file.write(line + "\n")
+                    self._file.flush()
+                if self.console_enabled and _LEVELS[normalized_level] >= _LEVELS[self.console_level]:
+                    rendered_fields = [
+                        f"{key}={json.dumps(record[key], ensure_ascii=False, separators=(',', ':')) if isinstance(record[key], (Mapping, list, tuple)) else record[key]}"
+                        for key in sorted(record)
+                        if key not in _CORE_FIELDS and key != "schema_version"
+                    ]
+                    self.console_stream.write(
+                        " ".join(
+                            [
+                                record["timestamp"],
+                                normalized_level,
+                                f"run={self.run_id}",
+                                f"stage={normalized_stage}",
+                                f"event={normalized_event}",
+                                f"status={record['status']}",
+                                f"elapsed_ms={record['elapsed_ms']}",
+                                *rendered_fields,
+                            ]
+                        )
+                        + "\n"
+                    )
+                    self.console_stream.flush()
+            except (OSError, TypeError, ValueError) as error:
+                raise AuthorRunLoggingError(f"Cannot write log event '{normalized_stage}/{normalized_event}': {error}") from error
+        return record
+
+    @contextmanager
+    def stage(
+        self,
+        stage: str,
+        *,
+        failure_level: str = "ERROR",
+        failure_status: str = "FAILED",
+        **fields: object,
+    ) -> Iterator[None]:
+        started_at = perf_counter()
+        self.emit(stage, "started", status="RUNNING", **fields)
+        try:
+            yield
+        except Exception as error:
+            self.emit(
+                stage,
+                "failed",
+                level=failure_level,
+                status=failure_status,
+                elapsed_ms=(perf_counter() - started_at) * 1000,
+                error_code=type(error).__name__,
+                exception_type=type(error).__name__,
+                error=_safe_text(str(error)),
+                **fields,
+            )
+            raise
+        else:
+            self.emit(
+                stage,
+                "completed",
+                status="COMPLETED",
+                elapsed_ms=(perf_counter() - started_at) * 1000,
+                **fields,
+            )
+
+    def close(self) -> None:
+        with self._lock:
+            if self._file is None:
+                return
+            try:
+                self._file.flush()
+                self._file.close()
+            except OSError as error:
+                raise AuthorRunLoggingError(f"Cannot close log file '{self.jsonl_path}': {error}") from error
+            finally:
+                self._file = None
+
+
+__all__ = ["AUTHOR_LOGGING_SCHEMA_VERSION", "AuthorRunLogger", "AuthorRunLoggingError"]

src/agents/research_plan_author/section_cache.py

@@ -0,0 +1,184 @@
+"""Versioned content-addressed reuse for validated Author sections."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from copy import deepcopy
+from datetime import datetime, timezone
+import hashlib
+import json
+import os
+from pathlib import Path
+from threading import RLock
+from typing import Any
+from uuid import uuid4
+
+
+SECTION_COMPOSITION_CACHE_SCHEMA_VERSION = "research_plan_author_section_cache_v1"
+SECTION_COMPOSITION_CACHE_REVISION = "4"
+_CACHE_MODES = {"disabled", "read_write", "read_only", "refresh"}
+
+
+def _setting(value: object, key: str, default: object = "") -> object:
+    if isinstance(value, Mapping):
+        return value.get(key, default)
+    return getattr(value, key, default)
+
+
+def _as_bool(value: object, default: bool) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return default
+    return str(value).strip().casefold() in {"1", "true", "yes", "on"}
+
+
+def _canonical(value: object) -> object:
+    if isinstance(value, Mapping):
+        return {str(key): _canonical(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))}
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [_canonical(item) for item in value]
+    if isinstance(value, Path):
+        return str(value)
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return value
+    return str(value)
+
+
+def canonical_json(value: object) -> str:
+    return json.dumps(_canonical(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def content_digest(value: object) -> str:
+    return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def section_cache_identity(
+    *,
+    preparation: Mapping[str, Any],
+    blueprint: Mapping[str, Any],
+    route: Mapping[str, Any],
+    blueprint_section: Mapping[str, Any],
+    source_registry: Mapping[str, Any],
+) -> dict[str, Any]:
+    """Capture every source-bounded input that can change a section result."""
+
+    source_bundle = preparation.get("source_bundle")
+    return {
+        "cache_schema_version": SECTION_COMPOSITION_CACHE_SCHEMA_VERSION,
+        "composer_revision": SECTION_COMPOSITION_CACHE_REVISION,
+        "source_bundle": deepcopy(dict(source_bundle)) if isinstance(source_bundle, Mapping) else {},
+        "blueprint_global_constraints": deepcopy(dict(blueprint.get("global_constraints") or {})),
+        "blueprint_argument_ledger": deepcopy(dict(blueprint.get("argument_ledger") or {})),
+        "route": deepcopy(dict(route)),
+        "blueprint_section": deepcopy(dict(blueprint_section)),
+        "source_registry": deepcopy(dict(source_registry)),
+    }
+
+
+class SectionCompositionCache:
+    """Store only complete, validated section payloads; cache failures are misses."""
+
+    def __init__(self, config: object | None = None) -> None:
+        self.enabled = _as_bool(_setting(config, "enabled", True), True)
+        requested_mode = str(_setting(config, "mode", "read_write") or "read_write").strip().casefold()
+        self.mode = requested_mode if requested_mode in _CACHE_MODES else "read_write"
+        if not self.enabled:
+            self.mode = "disabled"
+        root = str(_setting(config, "root", ".science/cache/research_plan_author/v1") or ".science/cache/research_plan_author/v1")
+        self.root = Path(root).expanduser().resolve()
+        self._lock = RLock()
+        self._stats = {"hits": 0, "misses": 0, "writes": 0, "corrupt": 0}
+
+    @property
+    def can_read(self) -> bool:
+        return self.mode not in {"disabled", "refresh"}
+
+    @property
+    def can_write(self) -> bool:
+        return self.mode in {"read_write", "refresh"}
+
+    def read(self, identity: Mapping[str, Any]) -> dict[str, Any] | None:
+        if not self.can_read:
+            return None
+        identity_digest = content_digest(identity)
+        try:
+            with self._lock:
+                with self._path_for(identity_digest).open("r", encoding="utf-8") as handle:
+                    envelope = json.load(handle)
+                if not isinstance(envelope, Mapping):
+                    self._stats["corrupt"] += 1
+                    return None
+                section = envelope.get("section")
+                if (
+                    envelope.get("schema_version") != SECTION_COMPOSITION_CACHE_SCHEMA_VERSION
+                    or envelope.get("identity_digest") != identity_digest
+                    or not isinstance(section, Mapping)
+                    or envelope.get("section_digest") != content_digest(section)
+                ):
+                    self._stats["corrupt"] += 1
+                    return None
+                self._stats["hits"] += 1
+                return deepcopy(dict(section))
+        except (OSError, ValueError, json.JSONDecodeError):
+            with self._lock:
+                self._stats["misses"] += 1
+            return None
+
+    def write(self, identity: Mapping[str, Any], section: Mapping[str, Any]) -> bool:
+        if not self.can_write:
+            return False
+        identity_digest = content_digest(identity)
+        safe_section = deepcopy(dict(section))
+        envelope = {
+            "schema_version": SECTION_COMPOSITION_CACHE_SCHEMA_VERSION,
+            "identity_digest": identity_digest,
+            "section_digest": content_digest(safe_section),
+            "created_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
+            "section": safe_section,
+        }
+        try:
+            with self._lock:
+                self._atomic_write_json(self._path_for(identity_digest), envelope)
+                self._stats["writes"] += 1
+            return True
+        except OSError:
+            return False
+
+    def summary(self) -> dict[str, Any]:
+        with self._lock:
+            return {
+                "schema_version": SECTION_COMPOSITION_CACHE_SCHEMA_VERSION,
+                "mode": self.mode,
+                "root": str(self.root),
+                **self._stats,
+            }
+
+    def _path_for(self, identity_digest: str) -> Path:
+        return self.root / "sections" / f"{identity_digest}.json"
+
+    @staticmethod
+    def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+        try:
+            with temporary_path.open("x", encoding="utf-8", newline="\n") as handle:
+                handle.write(canonical_json(payload))
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary_path, path)
+        finally:
+            if temporary_path.exists():
+                try:
+                    temporary_path.unlink()
+                except OSError:
+                    pass
+
+
+__all__ = [
+    "SECTION_COMPOSITION_CACHE_REVISION",
+    "SECTION_COMPOSITION_CACHE_SCHEMA_VERSION",
+    "SectionCompositionCache",
+    "content_digest",
+    "section_cache_identity",
+]