conjured 0.0.1__tar.gz

conjured-0.0.1/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: conjured
+Version: 0.0.1
+Summary: Generative pipeline creator for interactive fiction
+Author-email: Bryan <myemfar@gmail.com>
+License-Expression: MIT
+Requires-Python: >=3.11

conjured-0.0.1/conjured/core/__init__.py

@@ -0,0 +1,10 @@
+"""Shared utilities — no project-internal imports. Any layer can use these.
+
+Usage:
+    from conjured.core import get_logger, load_toml, PROJECT_ROOT
+"""
+
+from conjured.core.config import load_toml, get_nested, invalidate
+from conjured.core.logging import get_logger, configure_logging, compute_profile, check_budget, format_report
+from conjured.core.paths import PROJECT_ROOT, PACKAGE_ROOT, CONFIG_DIR, MODELS_DIR
+from conjured.core.utils import deep_merge

conjured-0.0.1/conjured/core/artifacts.py

@@ -0,0 +1,98 @@
+"""Artifact registry and runtime loader.
+
+Registry for compiled artifacts (pre-processed game config → optimized
+runtime data). The build tool (tools/compile.py) compiles artifacts.
+Handlers load them at runtime via load_artifact().
+
+Usage:
+    from conjured.core.artifacts import register_artifact, load_artifact
+
+    @register_artifact("entity_dict")
+    class EntityDict:
+        name = "entity_dict"
+        def source_hash(self) -> str: ...
+        def compile(self, output_dir: Path) -> None: ...
+        def load(self, compiled_dir: Path) -> Any: ...
+
+    # At runtime (in a handler):
+    data = load_artifact("entity_dict")
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from conjured.core.logging import get_logger
+from conjured.core.paths import PROJECT_ROOT
+
+logger = get_logger(__name__)
+
+COMPILED_DIR = PROJECT_ROOT / "compiled"
+
+
+@dataclass
+class ArtifactMeta:
+    """Registry entry for a compiled artifact."""
+    name: str
+    compiler: object  # has source_hash(), compile(), load() methods
+    _instance: Any = field(default=None, repr=False)
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+
+# The registry — populated by @register_artifact
+ARTIFACTS: dict[str, ArtifactMeta] = {}
+
+
+def register_artifact(name: str):
+    """Decorator that registers an artifact compiler class.
+
+    The class must implement:
+        name: str
+        source_hash() -> str
+        compile(output_dir: Path) -> None
+        load(compiled_dir: Path) -> Any
+    """
+    def decorator(cls):
+        ARTIFACTS[name] = ArtifactMeta(name=name, compiler=cls())
+        return cls
+    return decorator
+
+
+def load_artifact(name: str) -> Any:
+    """Load a compiled artifact, using a cached singleton.
+
+    Thread-safe via per-artifact Lock with double-checked locking.
+    Raises FileNotFoundError if artifact is not compiled.
+    """
+    if name not in ARTIFACTS:
+        raise ValueError(f"Unknown artifact: {name}. Registered: {list(ARTIFACTS.keys())}")
+
+    meta = ARTIFACTS[name]
+
+    if meta._instance is not None:
+        return meta._instance
+
+    with meta._lock:
+        if meta._instance is not None:
+            return meta._instance
+
+        artifact_dir = COMPILED_DIR / name
+        if not artifact_dir.exists():
+            raise FileNotFoundError(
+                f"Artifact '{name}' not compiled. Run: python -m conjured.tools.compile"
+            )
+
+        meta._instance = meta.compiler.load(COMPILED_DIR)
+        return meta._instance
+
+
+def invalidate(name: str | None = None) -> None:
+    """Clear cached artifact instance(s), forcing reload on next access."""
+    targets = [name] if name else list(ARTIFACTS.keys())
+    for n in targets:
+        if n in ARTIFACTS:
+            with ARTIFACTS[n]._lock:
+                ARTIFACTS[n]._instance = None

conjured-0.0.1/conjured/core/config.py

@@ -0,0 +1,53 @@
+"""Generic TOML config loading with caching.
+
+No engine, API, or game-specific logic. Any module can import this.
+
+Usage:
+    from conjured.core.config import load_toml, get_nested, invalidate
+
+    cfg = load_toml(Path("config/something.toml"))
+    val = get_nested(cfg, "section.key", default="fallback")
+"""
+
+import tomllib
+from pathlib import Path
+
+_cache: dict[Path, dict] = {}
+
+
+def load_toml(path: Path) -> dict:
+    """Load and cache a TOML file. Returns the parsed dict.
+
+    Cache is keyed by resolved path, so symlinks and relative paths
+    resolve to the same entry. Call invalidate() to clear.
+    """
+    key = path.resolve()
+    if key not in _cache:
+        with open(key, "rb") as f:
+            _cache[key] = tomllib.load(f)
+    return _cache[key]
+
+
+def get_nested(config: dict, dotted_key: str, default=None):
+    """Traverse a nested dict using dot-separated keys.
+
+    get_nested(cfg, "generation.temperature", default=0.7)
+    is equivalent to cfg.get("generation", {}).get("temperature", 0.7)
+    """
+    parts = dotted_key.split(".")
+    current = config
+    for part in parts[:-1]:
+        if not isinstance(current, dict):
+            return default
+        current = current.get(part, {})
+    if not isinstance(current, dict):
+        return default
+    return current.get(parts[-1], default)
+
+
+def invalidate(path: Path | None = None):
+    """Clear cached config. If path given, clear only that file. Otherwise clear all."""
+    if path is not None:
+        _cache.pop(path.resolve(), None)
+    else:
+        _cache.clear()

conjured-0.0.1/conjured/core/logging.py

@@ -0,0 +1,330 @@
+"""Structured logging, latency profiling, and timing capture.
+
+No engine, API, or game-specific logic. Any module can import this.
+
+Usage:
+    from conjured.core.logging import get_logger
+    logger = get_logger(__name__)
+    logger.info("event_name", key=value)
+
+Configure output format via RPCHAT_LOG_FORMAT env var:
+    "console" (default) — human-readable colored output
+    "json" — structured JSON lines (for production/parsing)
+
+Log files:
+    QA/{RPCHAT_QA_ENV}/logs/rpchat.log — JSON lines, rotates at 5 MB, keeps 50 files.
+    QA/{RPCHAT_QA_ENV}/turn-timings.jsonl — per-turn timing records.
+    RPCHAT_QA_ENV defaults to "default" if unset.
+
+Latency profiling:
+    from conjured.core.logging import compute_profile, check_budget, format_report
+    profile = compute_profile(list_of_timing_dicts)
+    report = format_report(profile, "archetype_companion", budget_ms=6000)
+"""
+
+import json
+import math
+import os
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from dotenv import load_dotenv
+import structlog
+
+load_dotenv()
+
+from conjured.core.paths import PROJECT_ROOT
+
+QA_ENV = os.environ.get("RPCHAT_QA_ENV", "default")
+_QA_DIR = PROJECT_ROOT / "QA"
+_LOG_DIR = _QA_DIR / QA_ENV / "logs"
+_TIMING_DIR = _QA_DIR / QA_ENV
+_TIMING_PATH = _TIMING_DIR / "turn-timings.jsonl"
+
+# Noisy third-party loggers — suppress to WARNING
+_QUIET_LOGGERS = [
+    "uvicorn.access",
+    "uvicorn.error",
+    "httpx",
+    "httpcore",
+    "sentence_transformers",
+    "transformers",
+    "onnxruntime",
+    "multipart",
+]
+
+
+def configure_logging():
+    """Set up structlog with console + rotating file output.
+
+    Console: human-readable or JSON (via RPCHAT_LOG_FORMAT).
+    File:    always JSON lines at QA/{env}/logs/rpchat.log, 5 MB per file, 50 kept (~250 MB max).
+    """
+    log_format = os.environ.get("RPCHAT_LOG_FORMAT", "console")
+
+    shared_processors = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+    ]
+
+    if log_format == "json":
+        console_renderer = structlog.processors.JSONRenderer()
+    else:
+        console_renderer = structlog.dev.ConsoleRenderer()
+
+    structlog.configure(
+        processors=[
+            *shared_processors,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    console_formatter = structlog.stdlib.ProcessorFormatter(
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            console_renderer,
+        ],
+    )
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(console_formatter)
+
+    _LOG_DIR.mkdir(parents=True, exist_ok=True)
+    log_path = _LOG_DIR / "rpchat.log"
+    file_formatter = structlog.stdlib.ProcessorFormatter(
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            structlog.processors.JSONRenderer(),
+        ],
+    )
+    file_handler = RotatingFileHandler(
+        str(log_path), maxBytes=5_000_000, backupCount=50, encoding="utf-8",
+    )
+    file_handler.setFormatter(file_formatter)
+
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.addHandler(console_handler)
+    root.addHandler(file_handler)
+    root.setLevel(logging.INFO)
+
+    for name in _QUIET_LOGGERS:
+        logging.getLogger(name).setLevel(logging.WARNING)
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    """Return a named structlog logger."""
+    return structlog.get_logger(name)
+
+
+# ---------------------------------------------------------------------------
+# Turn timing capture
+# ---------------------------------------------------------------------------
+
+def record_turn_timing(
+    pipeline_name: str,
+    handler_timings: dict,
+) -> None:
+    """Append a single timing record to the QA turn-timings JSONL file.
+
+    Called by the pipeline runner after each turn completes. Failure is
+    silent (warning log) — timing capture must never break gameplay.
+
+    Args:
+        pipeline_name: Name of the pipeline that ran (from config).
+        handler_timings: {handler_name: elapsed_ms} dict accumulated by the runner.
+    """
+    _logger = get_logger(__name__)
+    try:
+        total_ms = sum(v for v in handler_timings.values() if isinstance(v, (int, float)))
+        record = {
+            "ts": datetime.now(timezone.utc).isoformat(),
+            "pipeline": pipeline_name,
+            "total_ms": total_ms,
+            **handler_timings,
+        }
+        os.makedirs(_TIMING_DIR, exist_ok=True)
+        with open(_TIMING_PATH, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record) + "\n")
+    except Exception:
+        _logger.warning("qa_timing_write_failed", path=str(_TIMING_PATH), exc_info=True)
+
+
+# ---------------------------------------------------------------------------
+# Latency profiling
+# ---------------------------------------------------------------------------
+
+@dataclass
+class LatencyStats:
+    """Aggregated latency statistics for a single handler or total pipeline."""
+    mean: float = 0.0
+    p50: float = 0.0
+    p95: float = 0.0
+    p99: float = 0.0
+    min: float = 0.0
+    max: float = 0.0
+    count: int = 0
+
+
+def compute_handler_stats(values: list[float]) -> LatencyStats:
+    """Compute percentile statistics from a list of ms values."""
+    if not values:
+        return LatencyStats()
+
+    sorted_vals = sorted(values)
+    n = len(sorted_vals)
+
+    return LatencyStats(
+        mean=sum(sorted_vals) / n,
+        p50=_percentile(sorted_vals, 50),
+        p95=_percentile(sorted_vals, 95),
+        p99=_percentile(sorted_vals, 99),
+        min=sorted_vals[0],
+        max=sorted_vals[-1],
+        count=n,
+    )
+
+
+def compute_profile(timing_dicts: list[dict]) -> dict[str, LatencyStats]:
+    """Compute per-handler stats from a list of timing dicts.
+
+    Each dict has keys like "gate_ms", "identity_ms", "generation_ms", etc.
+    Keys ending in "_ms" are treated as handler timings. Also computes a
+    "total" entry from the sum of all handlers per dict.
+
+    Returns: {handler_name: LatencyStats, ..., "total": LatencyStats}
+    """
+    if not timing_dicts:
+        return {}
+
+    handler_values: dict[str, list[float]] = {}
+    totals: list[float] = []
+
+    for td in timing_dicts:
+        turn_total = 0.0
+        for key, val in td.items():
+            if not isinstance(val, (int, float)):
+                continue
+            name = key[:-3] if key.endswith("_ms") else key
+            handler_values.setdefault(name, []).append(float(val))
+            turn_total += float(val)
+        totals.append(turn_total)
+
+    profile = {name: compute_handler_stats(vals) for name, vals in handler_values.items()}
+    profile["total"] = compute_handler_stats(totals)
+
+    return profile
+
+
+def check_budget(total_ms: float, budget_ms: float) -> bool:
+    """Returns True if total_ms exceeds the budget."""
+    return total_ms > budget_ms
+
+
+def get_budget_for_pipeline(pipeline_name: str, budgets: dict) -> float | None:
+    """Look up the latency budget for a pipeline name.
+
+    Tries exact match first, then longest prefix match, then "default".
+    Config is a plain dict: {"archetype_companion": 6000, "archetype": 5000, "default": 8000}
+
+    Returns budget in ms, or None if no budget configured.
+    """
+    if not budgets:
+        return None
+
+    if pipeline_name in budgets:
+        return float(budgets[pipeline_name])
+
+    # Longest prefix match
+    best_match = None
+    best_len = 0
+    for key, val in budgets.items():
+        if key == "default":
+            continue
+        if pipeline_name.startswith(key) and len(key) > best_len:
+            best_match = val
+            best_len = len(key)
+
+    if best_match is not None:
+        return float(best_match)
+
+    if "default" in budgets:
+        return float(budgets["default"])
+
+    return None
+
+
+def format_report(
+    profile: dict[str, LatencyStats],
+    label: str,
+    budget_ms: float | None = None,
+) -> str:
+    """Format a latency profile as a text table."""
+    if not profile:
+        return f"  {label}: no data\n"
+
+    lines = []
+    budget_str = f" (budget: {budget_ms:.0f}ms)" if budget_ms else ""
+    total = profile.get("total")
+    total_str = f" — total p50: {total.p50:.0f}ms" if total else ""
+    lines.append(f"  {label}{budget_str}{total_str}")
+    lines.append(f"  {'Handler':<20} {'Mean':>7} {'P50':>7} {'P95':>7} {'P99':>7} {'Min':>7} {'Max':>7} {'N':>5}")
+    lines.append(f"  {'─' * 20} {'─' * 7} {'─' * 7} {'─' * 7} {'─' * 7} {'─' * 7} {'─' * 7} {'─' * 5}")
+
+    handler_order = sorted(k for k in profile if k != "total")
+    handler_order.append("total")
+
+    for handler in handler_order:
+        s = profile[handler]
+        marker = " *" if handler == "total" and budget_ms and s.p50 > budget_ms else ""
+        lines.append(
+            f"  {handler:<20} {s.mean:>6.0f}ms {s.p50:>6.0f}ms {s.p95:>6.0f}ms "
+            f"{s.p99:>6.0f}ms {s.min:>6.0f}ms {s.max:>6.0f}ms {s.count:>5}{marker}"
+        )
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def find_bottlenecks(
+    profiles: dict[str, dict[str, LatencyStats]],
+    top_n: int = 3,
+) -> list[tuple[str, str, float]]:
+    """Identify the top N slowest handlers across all pipelines.
+
+    Returns list of (pipeline, handler, p50_ms) tuples, sorted by p50 descending.
+    Excludes the "total" pseudo-entry.
+    """
+    entries = []
+    for pipeline, profile in profiles.items():
+        for handler, stats in profile.items():
+            if handler == "total" or stats.count == 0:
+                continue
+            entries.append((pipeline, handler, stats.p50))
+
+    entries.sort(key=lambda x: x[2], reverse=True)
+    return entries[:top_n]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _percentile(sorted_vals: list[float], pct: float) -> float:
+    """Compute percentile from a pre-sorted list using linear interpolation."""
+    if not sorted_vals:
+        return 0.0
+    if len(sorted_vals) == 1:
+        return sorted_vals[0]
+
+    rank = (pct / 100.0) * (len(sorted_vals) - 1)
+    lower = int(math.floor(rank))
+    upper = min(lower + 1, len(sorted_vals) - 1)
+    frac = rank - lower
+    return sorted_vals[lower] + frac * (sorted_vals[upper] - sorted_vals[lower])

conjured-0.0.1/conjured/core/parsing.py

@@ -0,0 +1,37 @@
+"""Generic text and JSON parsing utilities.
+
+No domain knowledge. Any project could use these.
+
+Usage:
+    from conjured.core.parsing import parse_json, strip_html_tags
+"""
+
+import json
+import re
+
+from conjured.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+_HTML_TAG_RE = re.compile(r"</?[a-zA-Z][^>]*>")
+
+
+def strip_html_tags(text: str) -> str:
+    """Remove HTML tags that LLMs sometimes inject into dialogue/narration."""
+    return _HTML_TAG_RE.sub("", text)
+
+
+def parse_json(raw: str) -> dict | None:
+    """Strip markdown fences, thinking blocks, and parse JSON. Returns dict or None."""
+    try:
+        clean = raw.strip()
+        if "<think>" in clean:
+            clean = re.sub(r'<think>.*?</think>\s*', '', clean, flags=re.DOTALL).strip()
+        if clean.startswith("```"):
+            clean = clean.split("```")[1]
+            if clean.startswith("json"):
+                clean = clean[4:]
+        return json.loads(clean.strip())
+    except Exception:
+        logger.exception("json_parse_failed")
+        return None

conjured-0.0.1/conjured/core/paths.py

@@ -0,0 +1,35 @@
+"""Path resolution — single source of truth for the engine package.
+
+Resolution order for CONFIG_DIR:
+    1. CONJURE_CONFIG_DIR env var (or .env)
+    2. conjured/config/ (ships with package — reference templates)
+
+Usage:
+    from conjured.core.paths import CONFIG_DIR, MODELS_DIR, PROJECT_ROOT
+"""
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# src/conjured/core/ → parent = src/conjured/
+PACKAGE_ROOT = Path(__file__).parent.parent
+
+# src/conjured/core/ → up to repo root
+PROJECT_ROOT = PACKAGE_ROOT.parent.parent
+
+# Config — inside the package (ships as reference templates)
+# Games/consumers override via CONJURE_CONFIG_DIR env var or .env
+CONFIG_DIR = Path(os.environ.get(
+    "CONJURE_CONFIG_DIR",
+    str(PACKAGE_ROOT / "config"),
+))
+
+# External paths — at repo root (not inside package)
+MODELS_DIR = Path(os.environ.get(
+    "CONJURE_MODELS_DIR",
+    str(PROJECT_ROOT / "models"),
+))

conjured-0.0.1/conjured/core/utils.py

@@ -0,0 +1,20 @@
+"""Shared utility functions used across layers."""
+
+from __future__ import annotations
+
+
+def deep_merge(base: dict, overlay: dict) -> dict:
+    """Recursively merge overlay into base. Overlay values win on conflict.
+
+    Lists are replaced (not appended). None values in overlay delete the key.
+    Returns a new dict — base is not mutated.
+    """
+    result = dict(base)
+    for key, value in overlay.items():
+        if value is None:
+            result.pop(key, None)
+        elif isinstance(value, dict) and isinstance(result.get(key), dict):
+            result[key] = deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result