pi-crew 0.9.47 → 0.9.49

package/skills/research/scripts/safe_io.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python3
+"""
+safe_io.py — SSRF-safe fetch guard + secret/PII redaction helper for the
+research/distill operational scripts (MEDIUM-3 + MEDIUM-4 hardening).
+
+Two functions:
+  - is_safe_url(url): SSRF guard. Returns True ONLY for http(s) URLs whose host
+    is not a private/loopback/link-local/metadata/reserved address. Rejects
+    non-http(s) schemes (file://, gopher://, dict://, ...). Optionally resolves
+    DNS and rejects if any A/AAAA record lands in a blocked range (DNS-rebind).
+  - redact_secrets(text): regex-based secret/PII redaction. Masks the VALUE of
+    private-key blocks, Authorization/Bearer headers, AWS keys, named
+    VAR=secret assignments, and common service tokens (sk_/pk_/xox_/gh*_),
+    keeping the finding TYPE + location intact
+    (e.g. "API_KEY=sk-abc123" -> "API_KEY=***REDACTED***").
+
+Stdlib only. Python 3.9+.
+
+Usage:
+    from safe_io import is_safe_url, redact_secrets
+    python3 safe_io.py --self-test    # exits 0 on success
+
+Imported by scripts that persist source content (redact_secrets) or fetch URLs
+(is_safe_url). Referenced from the 3 SKILL.md files so agents invoke it before
+persisting fetched content or fetching live URLs.
+"""
+import ipaddress
+import re
+import socket
+import sys
+from urllib.parse import urlparse
+
+
+REDACTED = "***REDACTED***"
+
+# Hostnames that ALWAYS resolve to loopback / cloud-metadata and must be
+# rejected even when resolve_dns=False (no network call). SSRF defense for the
+# common bypass: is_safe_url("http://localhost") would otherwise pass because
+# the hostname-only path skips DNS.
+_BLOCKED_HOSTNAMES = frozenset({
+    "localhost",
+    "localhost.localdomain",
+    "ip6-localhost",
+    "ip6-loopback",
+    "metadata",                    # cloud metadata shorthand
+    "metadata.google.internal",
+    "metadata.aws.internal",
+    "metadata.azure.com",
+})
+
+
+# --------------------------------------------------------------------------- #
+# redact_secrets — regex rules. Each preserves the type/key and blanks only
+# the secret VALUE. Order matters: blocks first, then headers, then inline.
+# --------------------------------------------------------------------------- #
+_SECRET_RULES = [
+    # 1. PEM/DER private-key blocks (keep the BEGIN/END markers, blank the body)
+    (
+        "private_key_block",
+        re.compile(
+            r"(-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY-----)"
+            r"[\s\S]*?"
+            r"(-----END (?:RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY-----)",
+            re.MULTILINE,
+        ),
+        lambda m: m.group(1) + "\n" + REDACTED + "\n-----END PRIVATE KEY-----",
+    ),
+    # 1b. Unterminated private-key BEGIN (leaked partial key — no END marker).
+    # Rule 1 above already handled any complete BEGIN/END block, so this only
+    # fires on partials. Matches BEGIN + base64-ish content to end of line.
+    (
+        "private_key_begin_only",
+        re.compile(
+            r"(-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY-----)"
+            r"([A-Za-z0-9+/=\s]+)$"
+        ),
+        lambda m: m.group(1) + "\n" + REDACTED,
+    ),
+    # 2. Authorization / Proxy-Authorization / X-Authorization header lines
+    (
+        "auth_header",
+        re.compile(
+            r"(?im)^(\s*(?:authorization|proxy-authorization|x-authorization)\s*:\s*"
+            r"(?:bearer|basic|token|jwt)?\s*)([^\r\n]+)"
+        ),
+        lambda m: m.group(1) + REDACTED,
+    ),
+    # 3. Inline "Bearer <token>" / "Basic <token>" anywhere
+    (
+        "bearer_inline",
+        re.compile(r"(?i)\b(bearer|basic)\s+([A-Za-z0-9\-._~+/]+=*)"),
+        lambda m: m.group(1) + " " + REDACTED,
+    ),
+    # 4. AWS access-key id (AKIA...) — the id itself is a credential
+    (
+        "aws_key_id",
+        re.compile(r"\b(AKIA[0-9A-Z]{16})\b"),
+        lambda m: REDACTED,
+    ),
+    # 4b. High-confidence credential names with SHORT values (password/passwd/pwd)
+    # — rule 5 below requires value length >=8, which misses weak short
+    # passwords. These names are almost always secrets, so redact any value.
+    (
+        "short_credential",
+        re.compile(r"(?i)\b(password|passwd|pwd)\s*[:=]\s*([^\s\"'#&|]+)"),
+        lambda m: m.group(1) + "=" + REDACTED,
+    ),
+    # 5. .env-style / JSON named assignments with a secret-like name:
+    #    keeps NAME= / "name": and blanks the value
+    (
+        "named_assignment",
+        re.compile(
+            r"(?i)([\"']?[A-Za-z0-9_\-\.]*(?:"
+            r"pass(?:word)?|secret|api[_-]?key|token|access[_-]?key|private[_-]?key|"
+            r"client[_-]?secret|auth(?:orization)?|credential|refresh[_-]?token|"
+            r"jwt|passwd"
+            r")[A-Za-z0-9_\-\.]*[\"']?\s*[:=]\s*)([\"']?)([^\"'\r\n#&|]{8,})"
+        ),
+        lambda m: m.group(1) + m.group(2) + REDACTED + m.group(2),
+    ),
+    # 6. Common service tokens standing alone (Stripe/OpenAI sk_/pk_, Slack xox, GitHub gh*)
+    (
+        "service_token",
+        re.compile(r"\b(sk|pk|xox[abp]|gh[opsu]|github_pat)_[A-Za-z0-9]{16,}\b"),
+        lambda m: REDACTED,
+    ),
+]
+def redact_secrets(text: str) -> str:
+    """Redact secret VALUES in *text*, keeping the type/location.
+
+    A second pass is safe: already-redacted spans do not match value patterns.
+    """
+    if not text:
+        return text
+    out = text
+    for _name, rx, build in _SECRET_RULES:
+        out = rx.sub(build, out)
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# is_safe_url — SSRF guard
+# --------------------------------------------------------------------------- #
+def _is_ip_literal(host: str) -> bool:
+    h = host.strip("[]")
+    try:
+        ipaddress.ip_address(h)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_blocked_ip(ip: str) -> bool:
+    """True if the IP is private/loopback/link-local/reserved/multicast/etc.
+
+    Fail-closed: an unparseable address is treated as blocked.
+    """
+    try:
+        addr = ipaddress.ip_address(ip.strip("[]"))
+    except ValueError:
+        return True
+    return (
+        addr.is_loopback
+        or addr.is_private
+        or addr.is_link_local
+        or addr.is_reserved
+        or addr.is_multicast
+        or addr.is_unspecified
+    )
+
+
+def is_safe_url(url: str, resolve_dns: bool = False) -> bool:
+    """SSRF guard. Returns True ONLY for http(s) URLs whose host is not a
+    private/loopback/link-local/metadata/reserved address.
+
+    - Rejects non-http(s) schemes (file://, gopher://, dict://, ftp://, ...).
+    - Rejects IP-literal hosts in blocked ranges (127.0.0.1, 169.254.169.254,
+      10/8, 172.16/12, 192.168/16, ::1, fc00::/7 ...).
+    - For hostnames, when resolve_dns=True: resolves DNS and rejects if ANY
+      A/AAAA record is in a blocked range (catches DNS-rebinding to 127.0.0.1).
+      Default resolve_dns=False does a hostname-only check (no network call) —
+      the DNS-rebinding guard is opt-in for callers that will actually fetch.
+    """
+    try:
+        parsed = urlparse(url)
+    except Exception:
+        return False
+    if parsed.scheme.lower() not in ("http", "https"):
+        return False
+    host = (parsed.hostname or "").lower()
+    if not host:
+        return False
+    # IP-literal host: check range directly (no DNS needed)
+    if _is_ip_literal(host):
+        return not _is_blocked_ip(host)
+    # Hostname: block well-known loopback / cloud-metadata aliases without DNS
+    if host in _BLOCKED_HOSTNAMES:
+        return False
+    # Hostname: optional DNS-rebinding check
+    if resolve_dns:
+        try:
+            infos = socket.getaddrinfo(host, None)
+        except socket.gaierror:
+            return False  # fail closed — cannot verify, refuse to fetch
+        for info in infos:
+            if _is_blocked_ip(info[4][0]):
+                return False
+    return True
+
+
+# --------------------------------------------------------------------------- #
+# self-test
+# --------------------------------------------------------------------------- #
+def self_test():
+    """Verify the SSRF guard rejects known-bad URLs and the redactor masks a
+    sample secret. Exits 0 on success, 1 on failure."""
+    ok = True
+
+    # --- SSRF: known-bad must REJECT (no DNS needed — IP literals) ---
+    bad_urls = [
+        "http://127.0.0.1/admin",
+        "http://169.254.169.254/latest/meta-data/",  # cloud metadata
+        "http://10.0.0.1/",
+        "http://192.168.1.1/",
+        "http://172.16.0.1/",
+        "http://[::1]/",
+        "http://0.0.0.0/",
+        "file:///etc/passwd",          # non-http scheme
+        "gopher://127.0.0.1:6379/",    # non-http scheme
+        "dict://localhost:11211/",     # non-http scheme
+        "ftp://internal/",             # non-http scheme
+        "http://localhost/admin",      # loopback HOSTNAME (not IP literal)
+        "http://metadata.google.internal/",  # cloud metadata hostname
+    ]
+    for u in bad_urls:
+        if is_safe_url(u):
+            print(f"  ✗ FAIL: bad URL allowed: {u}")
+            ok = False
+    # --- SSRF: known-good must ALLOW (resolve_dns=False — no network needed) ---
+    good_urls = [
+        "https://example.com/",
+        "http://example.org/path?q=1",
+        "https://arxiv.org/abs/2605.23899",
+        "https://8.8.8.8/",  # public IP literal
+    ]
+    for u in good_urls:
+        if not is_safe_url(u, resolve_dns=False):
+            print(f"  ✗ FAIL: good URL rejected: {u}")
+            ok = False
+
+    # --- redaction: sample secrets must be masked, type/location kept ---
+    samples = {
+        "API_KEY=sk-live-abcdef1234567890XYZ": "API_KEY=***REDACTED***",
+        'config = {"token": "ghp_abcdefghijklmnopqrstuvwxyz0123456789AB"}': "***REDACTED***",
+        "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig": "Authorization: ***REDACTED***",
+        "aws_access_key_id = AKIAIOSFODNN7EXAMPLE": "***REDACTED***",
+        "DATABASE_PASSWORD=hunter2-super-secret": "DATABASE_PASSWORD=***REDACTED***",
+    }
+    for inp, _expected_fragment in samples.items():
+        masked = redact_secrets(inp)
+        # The secret VALUE must no longer appear; the key/type must remain.
+        if "sk-live-abcdef1234567890XYZ" in masked and inp != masked:
+            print(f"  ✗ FAIL: secret value leaked: {masked}")
+            ok = False
+        if inp == masked:
+            print(f"  ✗ FAIL: nothing redacted in: {inp}")
+            ok = False
+        if REDACTED not in masked:
+            print(f"  ✗ FAIL: no redaction marker in: {masked}")
+            ok = False
+
+    # private-key block
+    pem = (
+        "-----BEGIN RSA PRIVATE KEY-----\n"
+        "MIIEpAIBAAKCAQEAabcd...supersecretkeymaterial\n"
+        "-----END RSA PRIVATE KEY-----"
+    )
+    masked_pem = redact_secrets(pem)
+    if "supersecretkeymaterial" in masked_pem or REDACTED not in masked_pem:
+        print(f"  ✗ FAIL: private-key body leaked: {masked_pem}")
+        ok = False
+
+    # partial private key WITHOUT an END marker (rule 1b)
+    partial_pem = "-----BEGIN PRIVATE KEY-----MIIEvQIBADANBsupersecret"
+    masked_partial = redact_secrets(partial_pem)
+    if "supersecret" in masked_partial or REDACTED not in masked_partial:
+        print(f"  ✗ FAIL: partial private-key leaked: {masked_partial}")
+        ok = False
+
+    # short password (rule 4b — rule 5 requires value >=8 chars)
+    masked_pw = redact_secrets("password=hunter2")
+    if "hunter2" in masked_pw or REDACTED not in masked_pw:
+        print(f"  ✗ FAIL: short password leaked: {masked_pw}")
+        ok = False
+
+    # benign text must pass through unchanged (no false positives)
+    benign = "The quick brown fox jumps over the lazy dog. https://example.com/path info@example.com"
+    if redact_secrets(benign) != benign:
+        print(f"  ✗ FAIL: false positive on benign text")
+        ok = False
+
+    print("safe_io self-test:", "PASS" if ok else "FAIL")
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    argv = sys.argv[1:]
+    if not argv or argv[0] in ("-h", "--help"):
+        print(__doc__)
+        sys.exit(0)
+    if argv[0] == "--self-test":
+        sys.exit(self_test())
+    print("Usage: safe_io.py --self-test", file=sys.stderr)
+    sys.exit(2)

package/skills/research/scripts/source_evaluator.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""
+source_evaluator.py — borrowed from Geek-skills-deep-research, ported for the
+research skill (F13: wired INTO the Agentic Protocol Step 2, never orphaned).
+
+3D filter on a source list:
+  1. Authority (domain class, peer-review, primary-vs-secondary)
+  2. Freshness (publication date / last-modified)
+  3. Primary-vs-secondary (original research vs aggregation)
+
+Accepts JSON input, emits a scored report. Default thresholds below.
+
+Stdlib only. Python 3.9+.
+
+Usage:
+    python3 source_evaluator.py <sources.json> [--min-authority 0.6] [--max-age-days 365]
+    python3 source_evaluator.py --self-test
+
+Exit codes:
+    0  ≥80% of sources pass thresholds
+    1  <80% of sources pass; see output
+    2  usage error
+"""
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+
+# MEDIUM-4: redact secret/PII values before persisting source-derived content.
+# Same-dir import (script dir is on sys.path[0]); graceful fallback if absent.
+try:
+    from safe_io import redact_secrets
+except ImportError:  # pragma: no cover - safe_io bundled alongside this script
+    redact_secrets = None
+
+
+DEFAULT_MIN_AUTHORITY = 0.6
+DEFAULT_MAX_AGE_DAYS = 365
+DEFAULT_ACCEPT_RATIO = 0.8
+
+
+def host_match(host: str, trusted: str) -> bool:
+    """Exact host or subdomain match — rejects lookalikes like evilgithub.com."""
+    return host == trusted or host.endswith("." + trusted)
+
+
+NEWS_DOMAINS = {"nytimes.com", "reuters.com", "bloomberg.com", "ft.com", "wsj.com"}
+BLOG_DOMAINS = {"medium.com", "substack.com", "wordpress.com"}
+
+
+def score_authority(url: str, source_meta: dict) -> float:
+    """
+    Heuristic authority score 0-1.
+    Primary author / official docs / peer-reviewed = 0.9-1.0
+    Engineering blog / canonical framework = 0.7-0.9
+    Medium-tier publication / reputable blog = 0.5-0.7
+    Forum / aggregation / personal blog = 0.2-0.5
+    Unknown = 0.4
+
+    NOTE: `primary` and `official` flags are self-declared hints from input JSON.
+    They boost scores but should NOT be the sole basis for trust — verify
+    provenance independently before relying on them for high-stakes decisions.
+    Domain matching uses exact-host-or-subdomain (no substring), so
+    `evilgithub.com` will NOT match `github.com`.
+    """
+    parsed = urlparse(url.lower())
+    host = (parsed.hostname or "").replace("www.", "")
+    # Authoritative classes
+    if source_meta.get("primary"):
+        return 1.0
+    if host.endswith((".edu", ".gov", ".ac.uk", ".ac.jp")):
+        return 0.95
+    if any(host_match(host, d) for d in ("arxiv.org", "nature.com", "science.org")):
+        return 0.95
+    if host_match(host, "github.io") or host_match(host, "github.com"):
+        # Official source code / docs — treat as authoritative if "official" flag
+        return 0.9 if source_meta.get("official") else 0.7
+    if host_match(host, "wikipedia.org"):
+        return 0.65  # secondary aggregator
+    # Class by exact domain allowlist (not substring — prevents spoofing)
+    if any(host_match(host, d) for d in NEWS_DOMAINS):
+        return 0.85
+    if any(host_match(host, d) for d in BLOG_DOMAINS):
+        return 0.5
+    return 0.4  # unknown
+
+
+def score_freshness(source_meta: dict, max_age_days: int, now: datetime) -> float:
+    """
+    Heuristic freshness score 0-1.
+    - < 30 days: 1.0
+    - < 90 days: 0.9
+    - < 180 days: 0.8
+    - < 365 days: 0.7
+    - > 365 days: 0.5
+    - no date: 0.5
+    """
+    date_str = source_meta.get("date") or source_meta.get("published") or source_meta.get("last_modified")
+    if not date_str:
+        return 0.5
+    try:
+        d = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
+    except ValueError:
+        try:
+            d = datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            return 0.5
+    age = (now - d.replace(tzinfo=timezone.utc)).days if d.tzinfo is None else (now - d).days
+    if age < 30:
+        return 1.0
+    if age < 90:
+        return 0.9
+    if age < 180:
+        return 0.8
+    if age < max_age_days:
+        return 0.7
+    return 0.5
+
+
+def score_primary(source_meta: dict) -> float:
+    """Heuristic primary-vs-secondary score 0-1."""
+    if source_meta.get("primary"):
+        return 1.0
+    if source_meta.get("secondary"):
+        return 0.3
+    return 0.6  # neutral
+
+
+def main(argv):
+    if not argv or argv[0] in ("-h", "--help"):
+        print(__doc__)
+        return 0
+    if argv[0] == "--self-test":
+        return self_test()
+    if len(argv) < 1:
+        print("Usage: source_evaluator.py <sources.json> [--min-authority 0.6] [--max-age-days 365]", file=sys.stderr)
+        return 2
+
+    sources_path = Path(argv[0])
+    min_authority = DEFAULT_MIN_AUTHORITY
+    max_age = DEFAULT_MAX_AGE_DAYS
+    accept_ratio = DEFAULT_ACCEPT_RATIO
+
+    i = 1
+    while i < len(argv):
+        if argv[i] == "--min-authority" and i + 1 < len(argv):
+            min_authority = float(argv[i + 1])
+            i += 2
+        elif argv[i] == "--max-age-days" and i + 1 < len(argv):
+            max_age = int(argv[i + 1])
+            i += 2
+        elif argv[i] == "--accept-ratio" and i + 1 < len(argv):
+            accept_ratio = float(argv[i + 1])
+            i += 2
+        else:
+            i += 1
+
+    data = json.loads(sources_path.read_text(encoding="utf-8"))
+    sources = data.get("sources", [])
+    now = datetime.now(timezone.utc)
+
+    scored = []
+    for s in sources:
+        url = s.get("url", "")
+        meta = {k: v for k, v in s.items() if k != "url"}
+        a = score_authority(url, meta)
+        f = score_freshness(meta, max_age, now)
+        p = score_primary(meta)
+        # Composite: 0.5 * authority + 0.3 * freshness + 0.2 * primary
+        composite = 0.5 * a + 0.3 * f + 0.2 * p
+        passes = a >= min_authority and f >= 0.5
+        scored.append({
+            "url": url,
+            "title": s.get("title", ""),
+            "authority": round(a, 3),
+            "freshness": round(f, 3),
+            "primary": round(p, 3),
+            "composite": round(composite, 3),
+            "passes": passes,
+        })
+
+    accepted = sum(1 for s in scored if s["passes"])
+    ratio = accepted / max(len(scored), 1)
+    ok = ratio >= accept_ratio
+
+    result = {
+        "ok": ok,
+        "thresholds": {
+            "min_authority": min_authority,
+            "max_age_days": max_age,
+            "accept_ratio": accept_ratio,
+        },
+        "stats": {
+            "total": len(scored),
+            "accepted": accepted,
+            "rejected": len(scored) - accepted,
+            "ratio": round(ratio, 3),
+        },
+        "sources": scored,
+    }
+
+    # MEDIUM-4: mask any secret/PII values that leaked into source metadata
+    # (titles, echoed content) before printing the report.
+    out = json.dumps(result, indent=2)
+    if redact_secrets:
+        out = redact_secrets(out)
+    print(out)
+    return 0 if ok else 1
+
+
+def self_test():
+    """Run a quick self-test on a fabricated source list."""
+    sources = {
+        "sources": [
+            {"url": "https://arxiv.org/abs/2605.23899", "date": "2026-06-01", "primary": True, "title": "SkillLens"},
+            {"url": "https://github.com/foo/bar", "official": True, "date": "2026-05-01", "title": "Official repo"},
+            {"url": "https://en.wikipedia.org/wiki/X", "date": "2026-07-01", "title": "Wikipedia article"},
+            {"url": "https://nytimes.com/article", "date": "2026-07-15", "title": "NYTimes article"},
+        ]
+    }
+    import tempfile
+    with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
+        json.dump(sources, f)
+        sp = f.name
+    code = main([sp])
+    Path(sp).unlink()
+    print(f"\nself-test: exit={code} (expected 0 — 4 sources, all above 0.6 authority)")
+    return code
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))