openrunner-sdk 2.7.0__tar.gz → 2.7.1__tar.gz

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.7.0
+Version: 2.7.1
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/openrunner/__init__.py

@@ -120,7 +120,7 @@ launch.from_run = _launch_from_run  # type: ignore[attr-defined]
 # openrunner.trace.patch_openai() syntax
 trace.patch_openai = _patch_openai  # type: ignore[attr-defined]
 
-__version__ = "2.7.0"
+__version__ = "2.7.1"
 
 logger = logging.getLogger("openrunner")
 

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/openrunner/cli.py

@@ -2391,13 +2391,18 @@ def session_setup() -> None:
 @click.option("--hours", "-h", default=24.0, help="Look back N hours (default: 24)")
 @click.option("--project", "-p", default=None, help="Target project (default: from config)")
 @click.option("--dry-run", is_flag=True, help="Show what would be synced without uploading")
-def session_sync(directory: str | None, hours: float, project: str | None, dry_run: bool) -> None:
+@click.option("--redact/--no-redact", default=None, help="Force redaction on/off (default: use config)")
+@click.option("--redact-mode", type=click.Choice(["regex", "ner"]), default=None, help="Redaction mode")
+@click.option("--public", "visibility", flag_value="public", help="Make session public")
+@click.option("--private", "visibility", flag_value="private", default=True, help="Keep session private (default)")
+def session_sync(directory: str | None, hours: float, project: str | None, dry_run: bool, redact: bool | None, redact_mode: str | None, visibility: str) -> None:
     """Sync AI sessions to OpenRunner.
 
     If DIRECTORY is given, scan that path for .jsonl/.json session files.
     Otherwise, scan default locations (~/.claude, ~/.codex, ~/.qwen-code).
 
     On first run, prompts for API key and project selection.
+    Redaction strips API keys, tokens, emails, passwords before upload.
     """
     from pathlib import Path
     from openrunner.session import discover_all_sessions, discover_in_directory, sync_all, get_session_config, interactive_setup
@@ -2427,7 +2432,7 @@ def session_sync(directory: str | None, hours: float, project: str | None, dry_r
         if dry_run:
             return
 
-    synced = sync_all(since_hours=hours, project=project, directory=Path(directory) if directory else None)
+    synced = sync_all(since_hours=hours, project=project, directory=Path(directory) if directory else None, redact=redact, redact_mode=redact_mode, visibility=visibility)
     if synced:
         click.echo(f"Synced {len(synced)} session(s) to OpenRunner.")
         for run_id in synced:

openrunner_sdk-2.7.1/openrunner/redact.py

@@ -0,0 +1,339 @@
+"""Session redaction — detect and mask PII/secrets before sync.
+
+Inspired by Dataiku's kiji-proxy (DeBERTa NER + synthetic replacement).
+
+Two modes:
+1. Regex-based (fast, no deps): API keys, tokens, passwords, IPs, emails, paths
+2. NER-based (accurate, needs transformers): full PII detection via DeBERTa
+
+Redaction can be configured at:
+- Client side: per-sync via `openrunner session sync --redact`
+- Organization level: org setting forces redaction for all members
+- User level: user setting in session_config.json
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Regex patterns for secrets and common PII
+# ---------------------------------------------------------------------------
+
+SECRET_PATTERNS: list[tuple[str, re.Pattern]] = [
+    # API keys / tokens (generic patterns)
+    ("API_KEY", re.compile(r"\b(sk-[a-zA-Z0-9\-_]{20,})\b")),  # OpenAI
+    ("API_KEY", re.compile(r"\b(or_[a-zA-Z0-9_\-]{20,})\b")),  # OpenRunner
+    ("API_KEY", re.compile(r"\b(ghp_[a-zA-Z0-9]{36,})\b")),  # GitHub PAT
+    ("API_KEY", re.compile(r"\b(gho_[a-zA-Z0-9]{36,})\b")),  # GitHub OAuth
+    ("API_KEY", re.compile(r"\b(github_pat_[a-zA-Z0-9_]{40,})\b")),  # GitHub fine-grained
+    ("API_KEY", re.compile(r"\b(pypi-[a-zA-Z0-9_\-]{50,})\b")),  # PyPI
+    ("API_KEY", re.compile(r"\b(npm_[a-zA-Z0-9]{30,})\b")),  # npm
+    ("API_KEY", re.compile(r"\b(xox[bsapr]-[a-zA-Z0-9\-]{10,})\b")),  # Slack
+    ("API_KEY", re.compile(r"\b(AKIA[0-9A-Z]{16})\b")),  # AWS access key
+    ("SECRET", re.compile(r"\b([a-zA-Z0-9/+=]{40})\b(?=.*(?:secret|SECRET))")),  # AWS secret
+    ("API_KEY", re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})\b")),  # Google API
+    ("TOKEN", re.compile(r"\b(eyJ[a-zA-Z0-9_\-]{20,}\.[a-zA-Z0-9_\-]{20,}\.[a-zA-Z0-9_\-]{20,})\b")),  # JWT
+    # Passwords in config/env
+    ("PASSWORD", re.compile(r"(?i)(?:password|passwd|pwd)\s*[=:]\s*['\"]?([^\s'\"]{6,})['\"]?")),
+    # Connection strings
+    ("CONNECTION_STRING", re.compile(r"(?i)((?:postgres|mysql|mongodb|redis)://[^\s'\"]+)")),
+    # Private keys
+    ("PRIVATE_KEY", re.compile(r"(-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----[^-]+-----END (?:RSA |EC |DSA )?PRIVATE KEY-----)", re.DOTALL)),
+    # Email addresses
+    ("EMAIL", re.compile(r"\b([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})\b")),
+    # IP addresses (non-localhost, non-docker)
+    ("IP_ADDRESS", re.compile(r"\b((?:(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d))\b")),
+    # Home directory paths (contain username)
+    ("PATH", re.compile(r"(/(?:home|Users)/[a-zA-Z0-9._\-]+)")),
+]
+
+# IPs to NOT redact (internal/docker/localhost)
+SAFE_IPS = {"127.0.0.1", "0.0.0.0", "localhost", "172.17.0.1", "172.18.0.1"}
+SAFE_IP_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "192.168.")
+
+# Emails to NOT redact
+SAFE_EMAILS = {"noreply@anthropic.com", "noreply@github.com"}
+
+
+def _is_safe_ip(ip: str) -> bool:
+    return ip in SAFE_IPS or any(ip.startswith(p) for p in SAFE_IP_PREFIXES)
+
+
+def _generate_replacement(label: str, original: str) -> str:
+    """Generate a deterministic replacement (same input -> same output)."""
+    # Use hash to generate consistent replacement
+    h = hashlib.sha256(original.encode()).hexdigest()[:8]
+
+    if label == "API_KEY":
+        return f"REDACTED_KEY_{h}"
+    elif label == "SECRET":
+        return f"REDACTED_SECRET_{h}"
+    elif label == "TOKEN":
+        return f"REDACTED_TOKEN_{h}"
+    elif label == "PASSWORD":
+        return f"REDACTED_PASS_{h}"
+    elif label == "CONNECTION_STRING":
+        # Keep protocol, redact rest
+        proto = original.split("://")[0] if "://" in original else "db"
+        return f"{proto}://REDACTED_{h}"
+    elif label == "PRIVATE_KEY":
+        return "-----BEGIN PRIVATE KEY-----\nREDACTED\n-----END PRIVATE KEY-----"
+    elif label == "EMAIL":
+        domain = original.split("@")[1] if "@" in original else "example.com"
+        return f"user_{h[:4]}@{domain}"
+    elif label == "IP_ADDRESS":
+        return f"x.x.x.{h[:2]}"
+    elif label == "PATH":
+        return f"/home/user_{h[:4]}"
+    else:
+        return f"[REDACTED:{label}]"
+
+
+# ---------------------------------------------------------------------------
+# Core redaction engine
+# ---------------------------------------------------------------------------
+
+
+class RedactionResult:
+    """Result of redacting text."""
+
+    def __init__(self, text: str, entities: list[dict], mapping: dict[str, str]):
+        self.text = text
+        self.entities = entities  # [{label, start, end, original, replacement}]
+        self.mapping = mapping  # original -> replacement (for restoration)
+
+    @property
+    def redacted_count(self) -> int:
+        return len(self.entities)
+
+
+def redact_text(text: str, mode: str = "regex") -> RedactionResult:
+    """Redact sensitive content from text.
+
+    Args:
+        text: Input text to redact.
+        mode: "regex" (fast, pattern-based) or "ner" (ML-based, needs transformers).
+
+    Returns:
+        RedactionResult with redacted text and metadata.
+    """
+    if mode == "ner":
+        return _redact_ner(text)
+    return _redact_regex(text)
+
+
+def _redact_regex(text: str) -> RedactionResult:
+    """Fast regex-based redaction for secrets and common PII."""
+    entities = []
+
+    for label, pattern in SECRET_PATTERNS:
+        for match in pattern.finditer(text):
+            original = match.group(1) if match.lastindex else match.group(0)
+            start = match.start(1) if match.lastindex else match.start(0)
+            end = match.end(1) if match.lastindex else match.end(0)
+
+            # Skip safe values
+            if label == "IP_ADDRESS" and _is_safe_ip(original):
+                continue
+            if label == "EMAIL" and original.lower() in SAFE_EMAILS:
+                continue
+            # Skip very short matches (likely false positives)
+            if len(original) < 6:
+                continue
+
+            entities.append({
+                "label": label,
+                "start": start,
+                "end": end,
+                "original": original,
+                "replacement": _generate_replacement(label, original),
+            })
+
+    # Deduplicate overlapping entities (keep longest)
+    entities.sort(key=lambda e: (e["start"], -(e["end"] - e["start"])))
+    deduped = []
+    last_end = -1
+    for e in entities:
+        if e["start"] >= last_end:
+            deduped.append(e)
+            last_end = e["end"]
+
+    # Apply replacements (end-to-start to preserve offsets)
+    result_text = text
+    mapping = {}
+    for e in reversed(deduped):
+        result_text = result_text[:e["start"]] + e["replacement"] + result_text[e["end"]:]
+        mapping[e["original"]] = e["replacement"]
+
+    return RedactionResult(result_text, deduped, mapping)
+
+
+def _redact_ner(text: str) -> RedactionResult:
+    """NER-based redaction using DeBERTa model (DataikuNLP/kiji-pii-model).
+
+    Falls back to regex if transformers not installed.
+    """
+    try:
+        from transformers import pipeline
+    except ImportError:
+        return _redact_regex(text)
+
+    # Load model (cached after first call)
+    global _ner_pipeline
+    if "_ner_pipeline" not in globals() or _ner_pipeline is None:
+        try:
+            _ner_pipeline = pipeline(
+                "token-classification",
+                model="DataikuNLP/kiji-pii-model-onnx",
+                aggregation_strategy="simple",
+            )
+        except Exception:
+            # Fall back to regex if model load fails
+            return _redact_regex(text)
+
+    # Run NER
+    try:
+        ner_results = _ner_pipeline(text[:10000])  # Cap at 10k chars
+    except Exception:
+        return _redact_regex(text)
+
+    entities = []
+    for ent in ner_results:
+        if ent.get("score", 0) < 0.25:
+            continue
+        label = ent.get("entity_group", ent.get("entity", "UNKNOWN"))
+        original = ent.get("word", "")
+        entities.append({
+            "label": label,
+            "start": ent["start"],
+            "end": ent["end"],
+            "original": original,
+            "replacement": _generate_replacement(label, original),
+        })
+
+    # Also run regex for secrets (NER won't catch API keys)
+    regex_result = _redact_regex(text)
+    # Merge: add regex entities that don't overlap with NER
+    for re_ent in regex_result.entities:
+        overlaps = any(
+            re_ent["start"] < e["end"] and re_ent["end"] > e["start"]
+            for e in entities
+        )
+        if not overlaps:
+            entities.append(re_ent)
+
+    entities.sort(key=lambda e: e["start"])
+
+    # Apply replacements
+    result_text = text
+    mapping = {}
+    for e in reversed(entities):
+        result_text = result_text[:e["start"]] + e["replacement"] + result_text[e["end"]:]
+        mapping[e["original"]] = e["replacement"]
+
+    return RedactionResult(result_text, entities, mapping)
+
+
+# ---------------------------------------------------------------------------
+# Session-level redaction
+# ---------------------------------------------------------------------------
+
+
+def redact_session(parsed: dict[str, Any], mode: str = "regex") -> dict[str, Any]:
+    """Redact a parsed session dict before sync.
+
+    Redacts:
+    - All message content (user + assistant)
+    - File paths (replace usernames)
+    - First message / summary
+
+    Returns a new dict (doesn't mutate input).
+    """
+    import copy
+    result = copy.deepcopy(parsed)
+
+    total_redacted = 0
+
+    # Redact messages
+    for msg in result.get("messages", []):
+        if msg.get("content"):
+            r = redact_text(msg["content"], mode=mode)
+            msg["content"] = r.text
+            total_redacted += r.redacted_count
+
+    # Redact first_message
+    if result.get("first_message"):
+        r = redact_text(result["first_message"], mode=mode)
+        result["first_message"] = r.text
+        total_redacted += r.redacted_count
+
+    # Redact summary
+    if result.get("summary"):
+        r = redact_text(result["summary"], mode=mode)
+        result["summary"] = r.text
+        total_redacted += r.redacted_count
+
+    # Redact file paths (just home dir usernames)
+    if result.get("files_touched"):
+        result["files_touched"] = [
+            re.sub(r"/(?:home|Users)/[^/]+", "/home/user", f)
+            for f in result["files_touched"]
+        ]
+
+    result["_redaction"] = {
+        "mode": mode,
+        "entities_redacted": total_redacted,
+    }
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Redaction policy config
+# ---------------------------------------------------------------------------
+
+
+class RedactionPolicy:
+    """Redaction policy: determines if/how to redact based on config."""
+
+    def __init__(
+        self,
+        enabled: bool = False,
+        mode: str = "regex",  # "regex" or "ner"
+        force: bool = False,  # org-level forced redaction
+    ):
+        self.enabled = enabled
+        self.mode = mode
+        self.force = force
+
+    @classmethod
+    def from_config(cls, config: dict) -> "RedactionPolicy":
+        """Load policy from session config or org settings."""
+        redaction = config.get("redaction", {})
+        return cls(
+            enabled=redaction.get("enabled", False),
+            mode=redaction.get("mode", "regex"),
+            force=redaction.get("force", False),
+        )
+
+    @classmethod
+    def from_org_settings(cls, org_settings: dict) -> "RedactionPolicy":
+        """Load policy from organization-level settings."""
+        if org_settings.get("force_session_redaction"):
+            return cls(enabled=True, mode=org_settings.get("redaction_mode", "regex"), force=True)
+        return cls(enabled=False)
+
+    def should_redact(self, user_choice: bool | None = None) -> bool:
+        """Determine if redaction should be applied.
+
+        Priority: org force > user explicit choice > config default.
+        """
+        if self.force:
+            return True
+        if user_choice is not None:
+            return user_choice
+        return self.enabled

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/openrunner/run.py

@@ -1188,15 +1188,7 @@ class Run:
         Returns:
             Path to the local artifact directory, or None on failure.
         """
-        if not self._client:
-            return None
-        return self._client.download_artifact(
-            run_id=self._run_id,
-            artifact_name=name,
-            dest_dir=dest_dir,
-            version=version,
-            alias=alias,
-        )
+        return self.use_artifact(name, version=version, alias=alias)
 
     def link_model(
         self,

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/openrunner/session.py

@@ -418,15 +418,52 @@ def sync_session_to_openrunner(
     project: str | None = None,
     api_key: str | None = None,
     base_url: str | None = None,
+    redact: bool | None = None,
+    redact_mode: str | None = None,
+    visibility: str = "private",
 ) -> str | None:
     """Upload a parsed session to OpenRunner as a run with notes.
 
+    Args:
+        redact: Force redaction on/off. None = use config/org policy.
+        redact_mode: "regex" (fast) or "ner" (ML-based, needs transformers).
+        visibility: "public" or "private".
+
     Returns the run ID on success, None on failure.
     """
     from openrunner.api_client import APIClient
+    from openrunner.redact import RedactionPolicy, redact_session
 
     # Load config (cascade: args > env > session_config > settings)
     config = get_session_config()
+
+    # Apply redaction policy (local config + server org policy)
+    policy = RedactionPolicy.from_config(config)
+
+    # Fetch server-side policy if not explicitly overridden
+    if redact is None and api_key and base_url:
+        try:
+            client_check = APIClient(base_url=base_url, api_key=api_key)
+            resp = client_check._request("get", "/users/me/settings/redaction")
+            if resp.status_code == 200:
+                server_policy = resp.json().get("effective", {})
+                if server_policy.get("forced") or server_policy.get("enabled"):
+                    policy = RedactionPolicy(
+                        enabled=True,
+                        mode=server_policy.get("mode", "regex"),
+                        force=server_policy.get("forced", False),
+                    )
+            client_check.close()
+        except Exception:
+            pass  # Offline or server doesn't support — use local config
+
+    if policy.should_redact(redact):
+        mode = redact_mode or policy.mode
+        parsed = redact_session(parsed, mode=mode)
+        redaction_info = parsed.get("_redaction", {})
+        if redaction_info.get("entities_redacted", 0) > 0:
+            logger.info(f"Redacted {redaction_info['entities_redacted']} sensitive entities ({mode} mode)")
+
     api_key = api_key or config.get("api_key")
     base_url = base_url or config.get("base_url")
 
@@ -457,11 +494,17 @@ def sync_session_to_openrunner(
             "tools_used": parsed.get("tools_used", []),
             "total_tokens": parsed.get("total_tokens", 0),
         },
-        "tags": [f"source:{source}", "ai-session"],
+        "tags": [f"source:{source}", "ai-session", f"visibility:{visibility}"],
         "notes": _format_session_notes(parsed),
         "state": "finished",
     }
 
+    # Add redaction metadata if applied
+    if parsed.get("_redaction"):
+        run_data["config"]["redacted"] = True
+        run_data["config"]["redaction_mode"] = parsed["_redaction"]["mode"]
+        run_data["config"]["entities_redacted"] = parsed["_redaction"]["entities_redacted"]
+
     result = client.create_run(run_data)
     if not result:
         logger.warning("Failed to create session run")
@@ -590,6 +633,9 @@ def sync_all(
     project: str | None = None,
     dry_run: bool = False,
     directory: Path | None = None,
+    redact: bool | None = None,
+    redact_mode: str | None = None,
+    visibility: str = "private",
 ) -> list[str]:
     """Discover and sync all new sessions. Returns list of synced run IDs."""
     state = _load_sync_state()
@@ -612,7 +658,7 @@ def sync_all(
             parsed = parse_generic_session(session_info["path"], session_info["source"])
 
         # Sync
-        run_id = sync_session_to_openrunner(parsed, project=project)
+        run_id = sync_session_to_openrunner(parsed, project=project, redact=redact, redact_mode=redact_mode, visibility=visibility)
         if run_id:
             state["synced"][h] = {
                 "run_id": run_id,

{openrunner_sdk-2.7.0 → openrunner_sdk-2.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.7.0"
+version = "2.7.1"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}