agentcloak 0.1.0__py3-none-any.whl

agentcloak/__init__.py

@@ -0,0 +1,7 @@
+"""agentcloak — Browser CLI toolchain for AI agents."""
+
+from importlib.metadata import version
+
+__version__ = version("agentcloak")
+
+__all__ = ["__version__"]

agentcloak/__main__.py

@@ -0,0 +1,5 @@
+"""Entry point for `python -m agentcloak`."""
+
+from agentcloak.cli.app import main
+
+main()

agentcloak/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""Site adapters — reusable site-specific automation."""
+
+from agentcloak.adapters.context import AdapterContext
+from agentcloak.adapters.executor import execute_adapter
+from agentcloak.adapters.registry import adapter, get_registry
+from agentcloak.adapters.types import AdapterEntry, AdapterMeta, Arg
+
+__all__ = [
+    "AdapterContext",
+    "AdapterEntry",
+    "AdapterMeta",
+    "Arg",
+    "adapter",
+    "execute_adapter",
+    "get_registry",
+]

agentcloak/adapters/analyzer.py

@@ -0,0 +1,252 @@
+"""API pattern recognition from captured network traffic."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, cast
+from urllib.parse import parse_qs, urlparse
+
+from agentcloak.core.types import Strategy
+
+if TYPE_CHECKING:
+    from agentcloak.core.capture import CaptureEntry
+
+__all__ = ["EndpointPattern", "PatternAnalyzer"]
+
+_UUID_RE = re.compile(
+    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I
+)
+_LONG_ID_RE = re.compile(r"/\d{4,}\b")
+_HEX_HASH_RE = re.compile(r"/[0-9a-f]{24,}", re.I)
+_DATE_RE = re.compile(r"/\d{4}-\d{2}-\d{2}")
+
+_AUTH_HEADERS = frozenset(
+    {
+        "authorization",
+        "x-csrf-token",
+        "x-api-key",
+        "api-key",
+        "x-auth-token",
+        "x-session-id",
+    }
+)
+
+_SKIP_DOMAINS = frozenset(
+    {
+        "fonts.googleapis.com",
+        "cdn.jsdelivr.net",
+        "www.google-analytics.com",
+        "www.googletagmanager.com",
+        "connect.facebook.net",
+    }
+)
+
+_PAGINATION_PARAMS = frozenset(
+    {"page", "offset", "limit", "cursor", "after", "before", "per_page", "pagesize"}
+)
+_FILTER_PARAMS = frozenset(
+    {"filter", "q", "query", "search", "status", "type", "category", "keyword"}
+)
+_SORT_PARAMS = frozenset({"sort", "order", "sort_by", "order_by"})
+
+
+@dataclass
+class EndpointPattern:
+    """Recognized API endpoint pattern."""
+
+    method: str
+    path: str
+    domain: str
+    call_count: int = 0
+    query_params: list[str] = field(default_factory=list[str])
+    status_codes: dict[int, int] = field(default_factory=dict[int, int])
+    auth_headers: list[str] = field(default_factory=list[str])
+    content_type: str = ""
+    category: str = "read"
+    request_schema: dict[str, Any] | None = None
+    response_schema: dict[str, Any] | None = None
+    strategy: Strategy = Strategy.PUBLIC
+    example_urls: list[str] = field(default_factory=list[str])
+
+
+def _normalize_path(path: str) -> str:
+    result = _UUID_RE.sub(":uuid", path)
+    result = _DATE_RE.sub("/:date", result)
+    result = _HEX_HASH_RE.sub("/:hash", result)
+    result = _LONG_ID_RE.sub("/:id", result)
+    return result
+
+
+def _extract_schema(obj: Any, max_depth: int = 2) -> Any:
+    if max_depth <= 0:
+        return type(obj).__name__
+    if isinstance(obj, dict):
+        return {
+            str(k): _extract_schema(v, max_depth - 1)
+            for k, v in cast("dict[str, Any]", obj).items()
+        }
+    if isinstance(obj, list):
+        if obj:
+            return [_extract_schema(obj[0], max_depth)]
+        return []
+    if isinstance(obj, bool):
+        return "bool"
+    if isinstance(obj, int):
+        return "int"
+    if isinstance(obj, float):
+        return "float"
+    if isinstance(obj, str):
+        return "str"
+    if obj is None:
+        return "null"
+    return type(obj).__name__
+
+
+def _infer_category(method: str, path: str) -> str:
+    p = path.lower()
+    if any(kw in p for kw in ("auth", "login", "token", "oauth", "signin", "signup")):
+        return "auth"
+    if any(kw in p for kw in ("search", "query", "find")):
+        return "search"
+    telemetry = ("log", "track", "event", "beacon", "metric", "telemetry")
+    if any(kw in p for kw in telemetry):
+        return "telemetry"
+    if method in ("GET", "HEAD", "OPTIONS"):
+        return "read"
+    return "write"
+
+
+def _infer_strategy(auth_headers: list[str]) -> Strategy:
+    if not auth_headers:
+        return Strategy.PUBLIC
+    lower = {h.lower() for h in auth_headers}
+    if "authorization" in lower:
+        return Strategy.HEADER
+    return Strategy.COOKIE
+
+
+class PatternAnalyzer:
+    """Analyze captured traffic to extract API endpoint patterns."""
+
+    def __init__(self, entries: list[CaptureEntry]) -> None:
+        self._entries = entries
+
+    def analyze(self) -> list[EndpointPattern]:
+        groups: dict[str, list[CaptureEntry]] = {}
+        for entry in self._entries:
+            parsed = urlparse(entry.url)
+            if parsed.hostname and parsed.hostname in _SKIP_DOMAINS:
+                continue
+            ct = entry.content_type.split(";", 1)[0].strip().lower()
+            if ct not in ("application/json", "text/json"):
+                continue
+            if entry.status == 0:
+                continue
+
+            normalized = _normalize_path(parsed.path)
+            key = f"{entry.method} {parsed.hostname} {normalized}"
+            groups.setdefault(key, []).append(entry)
+
+        patterns: list[EndpointPattern] = []
+        for key, group_entries in groups.items():
+            parts = key.split(" ", 2)
+            method = parts[0]
+            domain = parts[1]
+            path = parts[2]
+            pattern = self._build_pattern(method, domain, path, group_entries)
+            patterns.append(pattern)
+
+        patterns.sort(key=lambda p: p.call_count, reverse=True)
+        return patterns
+
+    def _build_pattern(
+        self,
+        method: str,
+        domain: str,
+        path: str,
+        entries: list[CaptureEntry],
+    ) -> EndpointPattern:
+        status_codes: dict[int, int] = {}
+        all_query_params: set[str] = set()
+        auth_headers_seen: set[str] = set()
+        content_types: list[str] = []
+        example_urls: list[str] = []
+        req_schemas: list[dict[str, Any]] = []
+        resp_schemas: list[dict[str, Any]] = []
+
+        for entry in entries:
+            status_codes[entry.status] = status_codes.get(entry.status, 0) + 1
+
+            parsed = urlparse(entry.url)
+            if parsed.query:
+                for part in parsed.query.split("&"):
+                    name = part.split("=", 1)[0]
+                    if name:
+                        all_query_params.add(name)
+
+            for header_name in entry.request_headers:
+                h = str(header_name)
+                if h.lower() in _AUTH_HEADERS:
+                    auth_headers_seen.add(h)
+
+            ct = entry.content_type.split(";", 1)[0].strip()
+            if ct:
+                content_types.append(ct)
+
+            if len(example_urls) < 3:
+                example_urls.append(entry.url)
+
+            if entry.request_body:
+                try:
+                    req_ct = next(
+                        (v for k, v in entry.request_headers.items() if k.lower() == "content-type"),
+                        "",
+                    ).split(";", 1)[0].strip().lower()
+                    if req_ct == "application/x-www-form-urlencoded":
+                        parsed_qs = parse_qs(entry.request_body, keep_blank_values=True)
+                        body_obj: Any = {
+                            k: v[0] if len(v) == 1 else v for k, v in parsed_qs.items()
+                        }
+                    else:
+                        body_obj = json.loads(entry.request_body)
+                    schema: Any = _extract_schema(body_obj)
+                    if isinstance(schema, dict):
+                        req_schemas.append(cast("dict[str, Any]", schema))
+                except (json.JSONDecodeError, ValueError):
+                    pass
+
+            if entry.response_body:
+                try:
+                    resp_obj: Any = json.loads(entry.response_body)
+                    resp_schema_val: Any = _extract_schema(resp_obj)
+                    if isinstance(resp_schema_val, dict):
+                        resp_schemas.append(cast("dict[str, Any]", resp_schema_val))
+                except (json.JSONDecodeError, ValueError):
+                    pass
+
+        auth_list = sorted(auth_headers_seen)
+        strategy = _infer_strategy(auth_list)
+        category = _infer_category(method, path)
+
+        req_schema = req_schemas[0] if req_schemas else None
+        resp_schema = resp_schemas[0] if resp_schemas else None
+
+        primary_ct = content_types[0] if content_types else ""
+
+        return EndpointPattern(
+            method=method,
+            path=path,
+            domain=domain,
+            call_count=len(entries),
+            query_params=sorted(all_query_params),
+            status_codes=status_codes,
+            auth_headers=auth_list,
+            content_type=primary_ct,
+            category=category,
+            request_schema=req_schema,
+            response_schema=resp_schema,
+            strategy=strategy,
+            example_urls=example_urls,
+        )

agentcloak/adapters/context.py

@@ -0,0 +1,76 @@
+"""AdapterContext — unified runtime interface for adapter handlers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from agentcloak.adapters.types import AdapterMeta
+    from agentcloak.browser.protocol import (
+        ActionResult,
+        BrowserContext,
+        NetworkRequest,
+    )
+    from agentcloak.browser.state import PageSnapshot
+
+__all__ = ["AdapterContext"]
+
+
+class AdapterContext:
+    """Wraps BrowserContext + parsed args for adapter execution."""
+
+    def __init__(
+        self,
+        *,
+        meta: AdapterMeta,
+        args: dict[str, Any],
+        browser: BrowserContext | None = None,
+    ) -> None:
+        self._meta = meta
+        self._args = args
+        self._browser = browser
+
+    @property
+    def meta(self) -> AdapterMeta:
+        return self._meta
+
+    @property
+    def args(self) -> dict[str, Any]:
+        return self._args
+
+    @property
+    def browser(self) -> BrowserContext:
+        if self._browser is None:
+            msg = "browser context not available for this adapter"
+            raise RuntimeError(msg)
+        return self._browser
+
+    @property
+    def has_browser(self) -> bool:
+        return self._browser is not None
+
+    # -- Convenience proxies to BrowserContext --
+
+    async def navigate(self, url: str, *, timeout: float = 30.0) -> ActionResult:
+        return await self.browser.navigate(url, timeout=timeout)
+
+    async def snapshot(self, *, mode: str = "accessible") -> PageSnapshot:
+        return await self.browser.snapshot(mode=mode)
+
+    async def action(self, kind: str, target: str, **kw: Any) -> ActionResult:
+        return await self.browser.action(kind, target, **kw)
+
+    async def evaluate(self, js: str) -> Any:
+        return await self.browser.evaluate(js)
+
+    async def network(
+        self, *, since: int | str = "last_action"
+    ) -> list[NetworkRequest]:
+        return await self.browser.network(since=since)
+
+    async def screenshot(self, *, full_page: bool = False) -> bytes:
+        return await self.browser.screenshot(full_page=full_page)
+
+    @property
+    def seq(self) -> int:
+        return self.browser.seq

agentcloak/adapters/discovery.py

@@ -0,0 +1,78 @@
+"""Adapter discovery — scan built-in and user directories for adapters."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+import structlog
+
+from agentcloak.adapters.registry import get_registry
+
+__all__ = ["discover_adapters"]
+
+log = structlog.get_logger()
+
+_USER_ADAPTER_DIR = Path.home() / ".config" / "agentcloak" / "adapters"
+
+
+def _import_module_from_path(name: str, path: Path) -> ModuleType | None:
+    """Import a Python module from a file path."""
+    spec = importlib.util.spec_from_file_location(name, path)
+    if spec is None or spec.loader is None:
+        return None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    try:
+        spec.loader.exec_module(module)  # type: ignore[union-attr]
+    except Exception:
+        log.warning("adapter.import_failed", path=str(path))
+        sys.modules.pop(name, None)
+        return None
+    return module
+
+
+def _discover_builtin() -> int:
+    """Import built-in adapters from adapters/sites/."""
+    count_before = len(get_registry())
+    builtin_module = "agentcloak.adapters.sites._example"
+    try:
+        if builtin_module in sys.modules:
+            importlib.reload(sys.modules[builtin_module])
+        else:
+            importlib.import_module(builtin_module)
+    except Exception:
+        log.warning("adapter.builtin_import_failed")
+    return len(get_registry()) - count_before
+
+
+def _discover_user() -> int:
+    """Scan user adapter directory for .py files."""
+    count_before = len(get_registry())
+    if not _USER_ADAPTER_DIR.is_dir():
+        return 0
+    for py_file in sorted(_USER_ADAPTER_DIR.glob("*.py")):
+        if py_file.name.startswith("_"):
+            continue
+        module_name = f"agentcloak_user_adapters.{py_file.stem}"
+        _import_module_from_path(module_name, py_file)
+    return len(get_registry()) - count_before
+
+
+def discover_adapters() -> dict[str, int]:
+    """Run full adapter discovery. Returns counts by source."""
+    builtin = _discover_builtin()
+    user = _discover_user()
+    log.debug(
+        "adapter.discovery_complete",
+        builtin=builtin,
+        user=user,
+        total=len(get_registry()),
+    )
+    return {"builtin": builtin, "user": user, "total": len(get_registry())}

agentcloak/adapters/executor.py

@@ -0,0 +1,66 @@
+"""Adapter execution — dispatches to function handler or pipeline engine."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+from agentcloak.adapters.context import AdapterContext
+from agentcloak.adapters.pipeline.engine import execute_pipeline
+from agentcloak.core.errors import AgentBrowserError
+
+if TYPE_CHECKING:
+    from agentcloak.adapters.types import AdapterEntry
+
+__all__ = ["execute_adapter"]
+
+log = structlog.get_logger()
+
+
+async def execute_adapter(
+    entry: AdapterEntry,
+    *,
+    args: dict[str, Any],
+    browser: Any | None = None,
+) -> list[dict[str, Any]]:
+    """Run an adapter and return its result rows."""
+    meta = entry.meta
+    log.info("adapter.execute", adapter=meta.full_name, strategy=meta.strategy)
+
+    for arg_def in meta.args:
+        if arg_def.name not in args and arg_def.default is not None:
+            args[arg_def.name] = arg_def.default
+
+    if meta.needs_browser and browser is None:
+        raise AgentBrowserError(
+            error="adapter_no_browser",
+            hint=f"Adapter '{meta.full_name}' requires a browser context "
+            f"(strategy={meta.strategy}).",
+            action="start a browser session first",
+        )
+
+    if meta.navigate_before and browser is not None:
+        log.debug("adapter.pre_navigate", url=meta.navigate_before)
+        await browser.navigate(meta.navigate_before)
+
+    if entry.is_pipeline:
+        if meta.pipeline is None:
+            raise RuntimeError(
+                f"Adapter '{meta.full_name}' marked as pipeline"
+                " but has no pipeline definition"
+            )
+        raw = await execute_pipeline(meta.pipeline, args=args, browser=browser)
+        if isinstance(raw, list):
+            return raw  # type: ignore[return-value]
+        return [raw] if raw is not None else []
+
+    if entry.handler is None:
+        raise AgentBrowserError(
+            error="adapter_no_handler",
+            hint=f"Adapter '{meta.full_name}' has neither pipeline nor handler.",
+            action="add a handler function or pipeline definition",
+        )
+
+    ctx = AdapterContext(meta=meta, args=args, browser=browser)
+    return await entry.handler(ctx)

agentcloak/adapters/generator.py

@@ -0,0 +1,156 @@
+"""Adapter code generation from API endpoint patterns."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from agentcloak.core.types import Strategy
+
+if TYPE_CHECKING:
+    from agentcloak.adapters.analyzer import EndpointPattern
+
+__all__ = ["generate_adapter", "generate_adapters"]
+
+_PARAM_RE = re.compile(r":(\w+)")
+
+
+def _path_params(path: str) -> list[str]:
+    return _PARAM_RE.findall(path)
+
+
+def _slugify(text: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_")
+    return slug or "unnamed"
+
+
+def _derive_name(pattern: EndpointPattern) -> str:
+    parts = [
+        p for p in pattern.path.strip("/").split("/")
+        if not p.startswith(":")
+        and p not in ("api", "v1", "v2", "v3", "v4")
+    ]
+    if parts:
+        return _slugify("_".join(parts[-2:]))
+    return _slugify(pattern.path)
+
+
+def _build_args_code(pattern: EndpointPattern) -> str:
+    lines: list[str] = []
+    for param in _path_params(pattern.path):
+        help_text = f"{param.title()} parameter"
+        lines.append(
+            f'        Arg("{param}", type=str,'
+            f' required=True, help="{help_text}"),'
+        )
+    for qp in pattern.query_params:
+        lines.append(
+            f'        Arg("{qp}", default=None, help="{qp} query parameter"),'
+        )
+    if not lines:
+        return ""
+    inner = "\n".join(lines)
+    return f"    args=[\n{inner}\n    ],\n"
+
+
+def _build_pipeline_code(pattern: EndpointPattern) -> str:
+    path = pattern.path
+    for param in _path_params(pattern.path):
+        path = path.replace(f":{param}", f"{{args.{param}}}")
+
+    url = f"https://{pattern.domain}{path}"
+
+    qp_parts: list[str] = []
+    for qp in pattern.query_params:
+        qp_parts.append(f"{qp}={{args.{qp}}}")
+    if qp_parts:
+        url += "?" + "&".join(qp_parts)
+
+    steps: list[str] = []
+
+    if pattern.strategy in (Strategy.COOKIE, Strategy.HEADER):
+        steps.append(f'        {{"navigate": "https://{pattern.domain}"}},')
+
+    if pattern.method == "GET":
+        steps.append(
+            f'        {{"evaluate": "fetch(\'{url}\', {{credentials: \'include\'}})'
+            f".then(r => r.json())\"}},"
+        )
+    else:
+        body_hint = "{}"
+        if pattern.request_schema:
+            keys = list(pattern.request_schema.keys())[:5]
+            pairs = ", ".join(f'\\"{k}\\": {{args.{k}}}' for k in keys)
+            body_hint = f"{{{pairs}}}"
+        steps.append(
+            f'        {{"evaluate": "fetch(\'{url}\', '
+            f"{{method: '{pattern.method}', "
+            f"credentials: 'include', "
+            f"headers: {{'Content-Type': 'application/json'}}, "
+            f"body: JSON.stringify({body_hint})"
+            f'}}).then(r => r.json())"}},',
+        )
+
+    return "\n".join(steps)
+
+
+def generate_adapter(
+    site: str,
+    pattern: EndpointPattern,
+    *,
+    name: str | None = None,
+) -> str:
+    """Generate Python adapter source code from an EndpointPattern."""
+    adapter_name = name or _derive_name(pattern)
+    func_name = f"{_slugify(site)}_{_slugify(adapter_name)}"
+    access = "read" if pattern.method in ("GET", "HEAD", "OPTIONS") else "write"
+
+    args_code = _build_args_code(pattern)
+    pipeline_code = _build_pipeline_code(pattern)
+
+    lines: list[str] = []
+    lines.append("@adapter(")
+    lines.append(f'    site="{site}",')
+    lines.append(f'    name="{adapter_name}",')
+    lines.append(f"    strategy=Strategy.{pattern.strategy.name},")
+    if pattern.domain:
+        lines.append(f'    domain="{pattern.domain}",')
+    lines.append(f'    description="{pattern.method} {pattern.path}",')
+    lines.append(f'    access="{access}",')
+    if args_code:
+        lines.append(args_code.rstrip())
+    lines.append("    pipeline=[")
+    lines.append(pipeline_code)
+    lines.append("    ],")
+    lines.append(")")
+    lines.append(f"def {func_name}() -> None:")
+    lines.append('    """Generated adapter — review before use."""')
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_adapters(
+    site: str,
+    patterns: list[EndpointPattern],
+) -> str:
+    """Generate a complete adapter module from multiple patterns."""
+    header_lines = [
+        f'"""Auto-generated adapters for {site} — review before use."""',
+        "",
+        "from __future__ import annotations",
+        "",
+        "from agentcloak.adapters.registry import adapter",
+        "from agentcloak.adapters.types import Arg",
+        "from agentcloak.core.types import Strategy",
+        "",
+    ]
+    header = "\n".join(header_lines) + "\n"
+
+    body_parts: list[str] = []
+    for pattern in patterns:
+        if pattern.category == "telemetry":
+            continue
+        body_parts.append(generate_adapter(site, pattern))
+
+    return header + "\n\n".join(body_parts)

agentcloak/adapters/pipeline/__init__.py

@@ -0,0 +1,6 @@
+"""Pipeline DSL — declarative data-flow adapter execution."""
+
+from agentcloak.adapters.pipeline.engine import execute_pipeline
+from agentcloak.adapters.pipeline.template import render, render_deep
+
+__all__ = ["execute_pipeline", "render", "render_deep"]