skeleton-key-http 0.1.0__py3-none-any.whl

skeleton_key/__init__.py

@@ -0,0 +1,92 @@
+"""skeleton_key — the canonical, declarative registry that formalizes the lockpicking model.
+
+Layered architecture (each layer's responsibility starts where the one below stops):
+
+    skeleton_key.transport      MECHANISM   the KEY: impersonate-only transport (TLS/JA3), cookie cache
+    resilient_fetch     RESILIENCE  identity rotation, egress Pool, browser fallback
+    skeleton_key (this) REGISTRY    the FORMAT: tumblers (detection rules) + picks (manifests)
+                                    + the derived tumbler->pick matrix + the drive loop
+
+The registry is language-agnostic YAML (registry/*.yaml, contract in registry/schema.json);
+this Python package is one engine for it. Detection is a Sigma-style match-DSL; picks are
+manifests bound to per-language handlers. The matrix is derived, never hand-written.
+
+Quick start:
+    from skeleton_key import open_door
+    result = open_door("https://www.cloudflare.com/cdn-cgi/trace")
+    print(result.opened, result.opened_by, result.resp.status_code)
+
+    # introspect the registry / matrix:
+    from skeleton_key import get_registry
+    reg = get_registry()
+    for tid, picks in reg.matrix.items():
+        print(tid, "->", [p.id for p in picks])
+"""
+
+from __future__ import annotations
+
+from .detect import (
+    KNOWN_OPS,
+    FormatError,
+    View,
+    detect,
+    evaluate,
+    is_blocked,
+    native_matcher,
+    validate_node,
+)
+from .engine import (
+    OpenResult,
+    Registry,
+    get_registry,
+    load_registry,
+    open_door,
+    resolve,
+)
+from .handlers import HANDLERS, handler
+from .schema import (
+    Attempt,
+    Category,
+    Cost,
+    Kind,
+    Pick,
+    PickContext,
+    Severity,
+    Tumbler,
+    TumblerHit,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # format types
+    "Category",
+    "Severity",
+    "Cost",
+    "Kind",
+    "Tumbler",
+    "Pick",
+    "TumblerHit",
+    "PickContext",
+    "Attempt",
+    # detection
+    "detect",
+    "is_blocked",
+    "evaluate",
+    "validate_node",
+    "View",
+    "native_matcher",
+    "KNOWN_OPS",
+    # engine
+    "Registry",
+    "OpenResult",
+    "load_registry",
+    "get_registry",
+    "resolve",
+    "open_door",
+    # handler binding
+    "HANDLERS",
+    "handler",
+    # errors
+    "FormatError",
+]

skeleton_key/__main__.py

@@ -0,0 +1,105 @@
+"""skeleton_key.__main__ — a thin command-line front door.
+
+    python -m skeleton_key <url> [--matrix] [--no-shims] [--no-browser] [-v]
+
+Drives the full honest ladder (the shim-aware ``open_door``: Key -> wrenches ->
+picks -> identity rotation -> browser, then alternate-route shims ONLY if the
+front door is confirmed stuck) and prints whether the door opened, what opened
+it, the HTTP status, and — when an alternate route was used — its fidelity
+label (source + fidelity class + freshness).
+
+Exit code is 0 when the door opened, non-zero when it is still locked, so the
+command composes in shell pipelines.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def _run(args: argparse.Namespace) -> int:
+    url = args.url
+
+    if args.matrix:
+        # Advanced: the registry's matrix-only loop (no shims, no rotation).
+        from skeleton_key import open_door as registry_open_door  # registry variant
+
+        result = registry_open_door(url, verbose=args.verbose)
+        status = getattr(result.resp, "status_code", None)
+        if result.opened:
+            print("opened: yes")
+            print(f"opened_by: {result.opened_by or 'front door (already open)'}")
+            print(f"status: {status}")
+            return 0
+        print("opened: no")
+        print("opened_by: -")
+        print(f"status: {status}")
+        return 2
+
+    # Default: the canonical shim-aware ladder.
+    from skeleton_key.shims import AllDoorsStuck, open_door
+
+    try:
+        door = open_door(
+            url,
+            use_browser=not args.no_browser,
+            allow_shims=not args.no_shims,
+            verbose=args.verbose,
+        )
+    except AllDoorsStuck as e:
+        print("opened: no")
+        print("opened_by: -")
+        print(f"reason: {e}")
+        return 2
+
+    status = getattr(door.resp, "status_code", None)
+    if door.via_shim and door.shim_result is not None:
+        opened_by = f"shim:{door.shim_result.shim}"
+    else:
+        opened_by = door.label()  # "front door (picks, <backend>)"
+
+    print("opened: yes")
+    print(f"opened_by: {opened_by}")
+    print(f"status: {status}")
+    if door.via_shim and door.shim_result is not None:
+        # Honest fidelity label: source + fidelity class + freshness.
+        print(f"fidelity: {door.shim_result.label()}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="Skeleton Key",
+        description="Open a door (URL) with an authentic browser key, honestly.",
+    )
+    parser.add_argument("url", help="the URL / door to open")
+    parser.add_argument(
+        "--matrix",
+        action="store_true",
+        help="use the registry's matrix-only loop (advanced; no rotation or shims)",
+    )
+    parser.add_argument(
+        "--no-shims",
+        action="store_true",
+        help="disable last-resort alternate-route shims (front door only)",
+    )
+    parser.add_argument(
+        "--no-browser",
+        action="store_true",
+        help="disable the browser fallback pass",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="print ladder progress to stderr"
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        return _run(args)
+    except KeyboardInterrupt:  # pragma: no cover
+        print("interrupted", file=sys.stderr)
+        return 130
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

skeleton_key/_paths.py

@@ -0,0 +1,45 @@
+"""skeleton_key._paths — cache-path parameterization.
+
+The engine persists a little state to disk: a cookie cache (so one solved
+challenge survives across CLI invocations and threads) and a persistent
+browser-profile dir (so a warmed identity sticks across browser fallbacks).
+
+Both are resolved through one helper so the package writes nowhere
+surprising and the location stays overridable:
+
+    cache_dir()  -> os.environ["SKELETON_KEY_CACHE"] if set,
+                    else platformdirs.user_cache_dir("skeleton_key").
+
+The directory is created on demand (exist_ok).
+"""
+
+from __future__ import annotations
+
+import os
+
+import platformdirs
+
+
+def cache_dir() -> str:
+    """The base cache directory for all on-disk state.
+
+    Honors the ``SKELETON_KEY_CACHE`` environment variable when set;
+    otherwise uses the per-user OS cache location.
+    """
+    base = os.environ.get("SKELETON_KEY_CACHE")
+    if not base:
+        base = platformdirs.user_cache_dir("skeleton_key")
+    os.makedirs(base, exist_ok=True)
+    return base
+
+
+def cookie_cache_path() -> str:
+    """Path to the per-registered-domain cookie cache JSON file."""
+    return os.path.join(cache_dir(), "skeleton_cookies.json")
+
+
+def browser_profile_dir() -> str:
+    """Path to the persistent browser profile directory (created on demand)."""
+    d = os.path.join(cache_dir(), "patchright-profile")
+    os.makedirs(d, exist_ok=True)
+    return d

skeleton_key/detect.py

@@ -0,0 +1,313 @@
+"""skeleton_key.detect — the predicate-DSL evaluator + unified tumbler detector.
+
+A tumbler's `match:` is a tree of primitive predicates combined by any/all/not. This is
+the SAME idea as Sigma / YARA: a language-neutral signature format with an engine in each
+language. The semantics below ARE the contract — any reimplementation (Rust, Go, JS) that
+evaluates a match-tree the same way is interoperable with the YAML registry verbatim.
+
+A match-node is a dict carrying exactly ONE key (an operator or a primitive):
+
+  Combinators
+    all:  [node, node, ...]      every child must hold        (AND)
+    any:  [node, node, ...]      at least one child holds     (OR)
+    not:  node                   the child must NOT hold      (NOT)
+
+  Primitives (evaluated against a response View — status, headers, body)
+    status:           [403, 429]         status code in this set (int or list)
+    status_range:     [500, 599]         low <= status <= high (inclusive)
+    header_present:   ["cf-mitigated"]   any of these header names present
+    header_absent:    ["x-foo"]          none of these header names present
+    header_equals:    {server: cloudflare}     header value == (case-insensitive)
+    header_contains:  {set-cookie: "datadome="} header value contains substring (ci)
+    body_contains:    ["just a moment"]  any substring present in body (case-insensitive)
+    body_regex:       "ray id:\\s*[0-9a-f]"     regex search over body (re.I)
+    body_len_lt:      1500                len(body bytes) < N
+    body_len_gte:     1500                len(body bytes) >= N
+    content_truncated: 0.6               body shorter than content-length * ratio
+    host_suffix:      ["reddit.com"]     response URL's host == / endswith any of these
+    url_contains:     ["/api/"]          response URL contains any of these substrings
+    native:           "fn_name"          ESCAPE HATCH -> a registered Python matcher.
+                                         Non-portable; use only for irreducible heuristics.
+
+Header names are matched case-insensitively (the View lowercases them). Body text is the
+first 64 KiB decoded utf-8 (replace); body_contains and body_regex are case-insensitive.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from urllib.parse import urlparse
+
+from .schema import Tumbler, TumblerHit
+
+_BODY_CAP = 65536  # first 64 KiB is plenty for any wall's interstitial/markers
+
+
+class FormatError(ValueError):
+    """A registry/match-tree that does not conform to the format. Raised at load time."""
+
+
+# ---------------------------------------------------------------------------
+# native matcher escape hatch — register a Python predicate by name
+# ---------------------------------------------------------------------------
+NATIVE_MATCHERS: dict[str, Callable[[View], bool]] = {}
+
+
+def native_matcher(name: str):
+    """Decorator: register a non-portable Python predicate for `match: {native: name}`."""
+
+    def deco(fn: Callable[[View], bool]):
+        NATIVE_MATCHERS[name] = fn
+        return fn
+
+    return deco
+
+
+# ---------------------------------------------------------------------------
+# View — a normalized, cheap-to-query snapshot of a Resp
+# ---------------------------------------------------------------------------
+class View:
+    """Normalized response facts the predicates read. Built once per detect() call."""
+
+    __slots__ = ("status", "headers", "raw_len", "_text", "_lower", "content_length", "url", "host")
+
+    def __init__(self, status: int, headers: dict, content: bytes, url: str = ""):
+        self.status = int(status)
+        # headers arrive lowercased from skeleton_key.transport; normalize defensively anyway.
+        self.headers = {str(k).lower(): v for k, v in (headers or {}).items()}
+        body = content or b""
+        self.raw_len = len(body)
+        self._text = body[:_BODY_CAP].decode("utf-8", "replace")
+        self._lower = self._text.lower()
+        cl = self.headers.get("content-length")
+        self.content_length = int(cl) if (cl and str(cl).isdigit()) else None
+        self.url = url or ""
+        self.host = (urlparse(self.url).hostname or "").lower()
+
+    @property
+    def text(self) -> str:
+        return self._text
+
+    @property
+    def lower(self) -> str:
+        return self._lower
+
+    @classmethod
+    def of(cls, resp) -> View:
+        return cls(
+            resp.status_code,
+            dict(getattr(resp, "headers", {}) or {}),
+            resp.content or b"",
+            getattr(resp, "url", "") or "",
+        )
+
+
+# ---------------------------------------------------------------------------
+# primitive evaluators — each takes (value_from_yaml, View) -> bool
+# ---------------------------------------------------------------------------
+def _as_list(v):
+    return v if isinstance(v, (list, tuple)) else [v]
+
+
+def _p_status(v, view: View) -> bool:
+    return view.status in {int(x) for x in _as_list(v)}
+
+
+def _p_status_range(v, view: View) -> bool:
+    lo, hi = int(v[0]), int(v[1])
+    return lo <= view.status <= hi
+
+
+def _p_header_present(v, view: View) -> bool:
+    return any(name.lower() in view.headers for name in _as_list(v))
+
+
+def _p_header_absent(v, view: View) -> bool:
+    return all(name.lower() not in view.headers for name in _as_list(v))
+
+
+def _p_header_equals(v, view: View) -> bool:
+    return all(
+        (view.headers.get(k.lower(), "") or "").lower() == str(val).lower() for k, val in v.items()
+    )
+
+
+def _p_header_contains(v, view: View) -> bool:
+    return all(
+        str(val).lower() in (view.headers.get(k.lower(), "") or "").lower() for k, val in v.items()
+    )
+
+
+def _p_body_contains(v, view: View) -> bool:
+    return any(str(s).lower() in view.lower for s in _as_list(v))
+
+
+_regex_cache: dict[str, re.Pattern] = {}
+
+
+def _p_body_regex(v, view: View) -> bool:
+    pat = _regex_cache.get(v)
+    if pat is None:
+        pat = _regex_cache[v] = re.compile(v, re.I)
+    return bool(pat.search(view.text))
+
+
+def _p_body_len_lt(v, view: View) -> bool:
+    return view.raw_len < int(v)
+
+
+def _p_body_len_gte(v, view: View) -> bool:
+    return view.raw_len >= int(v)
+
+
+def _p_content_truncated(v, view: View) -> bool:
+    return view.content_length is not None and view.raw_len < view.content_length * float(v)
+
+
+def _p_host_suffix(v, view: View) -> bool:
+    return any(view.host == s.lower() or view.host.endswith("." + s.lower()) for s in _as_list(v))
+
+
+def _p_url_contains(v, view: View) -> bool:
+    return any(str(s).lower() in view.url.lower() for s in _as_list(v))
+
+
+def _p_native(v, view: View) -> bool:
+    fn = NATIVE_MATCHERS.get(v)
+    if fn is None:
+        raise FormatError(f"native matcher {v!r} is not registered")
+    return bool(fn(view))
+
+
+_PRIMITIVES: dict[str, Callable] = {
+    "status": _p_status,
+    "status_range": _p_status_range,
+    "header_present": _p_header_present,
+    "header_absent": _p_header_absent,
+    "header_equals": _p_header_equals,
+    "header_contains": _p_header_contains,
+    "body_contains": _p_body_contains,
+    "body_regex": _p_body_regex,
+    "body_len_lt": _p_body_len_lt,
+    "body_len_gte": _p_body_len_gte,
+    "content_truncated": _p_content_truncated,
+    "host_suffix": _p_host_suffix,
+    "url_contains": _p_url_contains,
+    "native": _p_native,
+}
+_COMBINATORS = {"all", "any", "not"}
+KNOWN_OPS = set(_PRIMITIVES) | _COMBINATORS
+
+
+def evaluate(node, view: View) -> bool:
+    """Evaluate one match-node against a View. Raises FormatError on a malformed node."""
+    if not isinstance(node, dict) or len(node) != 1:
+        raise FormatError(f"match-node must be a dict with exactly one key, got: {node!r}")
+    op, val = next(iter(node.items()))
+    if op == "all":
+        return all(evaluate(child, view) for child in val)
+    if op == "any":
+        return any(evaluate(child, view) for child in val)
+    if op == "not":
+        return not evaluate(val, view)
+    fn = _PRIMITIVES.get(op)
+    if fn is None:
+        raise FormatError(f"unknown predicate {op!r} (known: {sorted(KNOWN_OPS)})")
+    return fn(val, view)
+
+
+def _check(cond, msg) -> None:
+    if not cond:
+        raise FormatError(msg)
+
+
+def _str_or_strlist(v) -> bool:
+    return isinstance(v, str) or (
+        isinstance(v, (list, tuple)) and bool(v) and all(isinstance(x, str) for x in v)
+    )
+
+
+def _validate_primitive(op, val) -> None:
+    """Type-check a primitive's value SHAPE so a malformed rule fails at LOAD, not mid-detect."""
+    if op == "status":
+        _check(
+            isinstance(val, int)
+            or (isinstance(val, (list, tuple)) and val and all(isinstance(x, int) for x in val)),
+            f"{op!r} needs an int or non-empty list of ints, got {val!r}",
+        )
+    elif op == "status_range":
+        _check(
+            isinstance(val, (list, tuple))
+            and len(val) == 2
+            and all(isinstance(x, int) for x in val),
+            f"status_range needs [low, high] ints, got {val!r}",
+        )
+    elif op in ("header_present", "header_absent", "body_contains", "host_suffix", "url_contains"):
+        _check(
+            _str_or_strlist(val), f"{op!r} needs a string or non-empty list of strings, got {val!r}"
+        )
+    elif op in ("header_equals", "header_contains"):
+        _check(
+            isinstance(val, dict)
+            and val
+            and all(isinstance(k, str) and isinstance(v, str) for k, v in val.items()),
+            f"{op!r} needs a non-empty string->string mapping, got {val!r}",
+        )
+    elif op in ("body_regex", "native"):
+        _check(isinstance(val, str) and val, f"{op!r} needs a non-empty string, got {val!r}")
+        if op == "body_regex":
+            try:
+                re.compile(val)
+            except re.error as e:
+                raise FormatError(f"body_regex is not a valid regex ({e}): {val!r}") from e
+    elif op in ("body_len_lt", "body_len_gte"):
+        _check(
+            isinstance(val, int) and not isinstance(val, bool), f"{op!r} needs an int, got {val!r}"
+        )
+    elif op == "content_truncated":
+        _check(
+            isinstance(val, (int, float)) and not isinstance(val, bool) and 0 < float(val) <= 1,
+            f"content_truncated needs a number in (0, 1], got {val!r}",
+        )
+
+
+def validate_node(node) -> None:
+    """Static check of a match-tree (no View needed). Validates structure AND primitive value
+    shapes, so a malformed registry raises FormatError at load instead of crashing at detect."""
+    if not isinstance(node, dict) or len(node) != 1:
+        raise FormatError(f"match-node must be a dict with exactly one key, got: {node!r}")
+    op, val = next(iter(node.items()))
+    if op in ("all", "any"):
+        if not isinstance(val, list) or not val:
+            raise FormatError(f"{op!r} needs a non-empty list of nodes")
+        for child in val:
+            validate_node(child)
+    elif op == "not":
+        validate_node(val)
+    elif op in _PRIMITIVES:
+        _validate_primitive(op, val)
+    else:
+        raise FormatError(f"unknown predicate {op!r} (known: {sorted(KNOWN_OPS)})")
+
+
+# ---------------------------------------------------------------------------
+# detection — run tumblers (severity-ordered) against a response
+# ---------------------------------------------------------------------------
+def detect(resp, tumblers: list[Tumbler]) -> TumblerHit | None:
+    """Return the highest-severity TumblerHit that fires on `resp`, or None if open.
+
+    `tumblers` should already be severity-ordered (engine sorts at load). The first match
+    wins, so a specific high-severity wall (e.g. a Cloudflare challenge 403) takes priority
+    over a generic low-severity rule (e.g. bare-403 reputation) when both could match."""
+    view = View.of(resp)
+    for t in tumblers:
+        if evaluate(t.match, view):
+            return TumblerHit(tumbler=t, reason=f"{t.category.value}:{t.id}")
+    return None
+
+
+def is_blocked(resp, tumblers: list[Tumbler]) -> tuple[bool, str]:
+    """Compatibility shim mirroring the legacy is_blocked() API, but registry-driven."""
+    hit = detect(resp, tumblers)
+    return (True, hit.reason) if hit else (False, "ok")