ghimera 0.3.0__py3-none-any.whl

chimera/__init__.py

@@ -0,0 +1,11 @@
+"""Legacy root facade; all objects belong to the single ghimera implementation.
+
+Nested imports must migrate to ghimera. This facade does not load duplicate
+submodules, install import hooks, or change saved configuration/result schemas.
+"""
+
+from ghimera import Collector, GhimeraConfig, Goal, GoalLoop, Harvest, Scope
+
+ChimeraConfig = GhimeraConfig
+
+__all__ = ["ChimeraConfig", "Collector", "Goal", "GoalLoop", "Harvest", "Scope"]

chimera/__main__.py

@@ -0,0 +1,5 @@
+"""Legacy module command using the same explicit ghimera entry point."""
+
+from ghimera.command import main
+
+raise SystemExit(main())

ghimera/__init__.py

@@ -0,0 +1,8 @@
+"""Ghimera core: no network, model, registry, or service is constructed on import."""
+
+from ghimera.collector import Collector
+from ghimera.config import GhimeraConfig
+from ghimera.loop import GoalLoop
+from ghimera.models import Goal, Harvest, Scope
+
+__all__ = ["GhimeraConfig", "Collector", "Goal", "GoalLoop", "Harvest", "Scope"]

ghimera/__main__.py

@@ -0,0 +1,5 @@
+"""Standalone entry point; services and credentials are selected by the caller."""
+
+from ghimera.command import main
+
+raise SystemExit(main())

ghimera/browser.py

@@ -0,0 +1,363 @@
+"""Networkless browser worker; the parent exclusively owns every HTTP/Tor fetch."""
+
+import asyncio
+import hashlib
+import os
+import shutil
+import signal
+import sys
+import tempfile
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Protocol
+
+from ghimera.browser_config import BrowserConfig
+from ghimera.browser_types import (
+    RenderResource,
+    RenderResult,
+    ResourceRequest,
+    ResourceResponse,
+    WorkerResult,
+)
+from ghimera.config import GhimeraConfig
+from ghimera.models import Page, Record, Scope
+from ghimera.passive_worker import private_directory, read_bounded
+from ghimera.refusals import GhimeraRefused, RefusalCode
+from ghimera.response import REDIRECT_STATUSES, redirect_target
+
+
+class ResourceFetcher(Protocol):
+    async def fetch(self, url: str) -> Page: ...
+
+
+class PageRenderer(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    def validate_config(self, config: GhimeraConfig) -> None: ...
+
+    async def render(
+        self,
+        page: Page,
+        scope: Scope,
+        resources: ResourceFetcher,
+        *,
+        timeout_seconds: float,
+    ) -> RenderResult: ...
+
+
+class BrowserRequest(Record):
+    config: BrowserConfig
+    page: Page
+    user_agent: str
+    parent_network_namespace: str
+
+
+class IsolatedBrowserRenderer:
+    name = "patchright_isolated"
+
+    def __init__(self, config: GhimeraConfig) -> None:
+        if config.browser is None:
+            raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+        self.config, self._user_agent = config.browser, config.user_agent
+        self._redirect_limit = config.http.max_redirects if config.http is not None else 0
+        try:
+            if version("patchright") != "1.63.0":
+                raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+            self._check_artifacts()
+        except (OSError, PackageNotFoundError):
+            raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT) from None
+        self._slots = asyncio.Semaphore(self.config.max_workers)
+
+    def _check_artifacts(self) -> None:
+        for path in (self.config.executable, self.config.isolator):
+            if not path.is_file() or not os.access(path, os.X_OK):
+                raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+        with self.config.executable.open("rb") as stream:
+            if hashlib.file_digest(stream, "sha256").hexdigest() != self.config.executable_sha256:
+                raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+
+    def validate_config(self, config: GhimeraConfig) -> None:
+        if (
+            config.browser != self.config
+            or config.user_agent != self._user_agent
+            or (config.http.max_redirects if config.http is not None else 0) != self._redirect_limit
+        ):
+            raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+
+    async def render(
+        self,
+        page: Page,
+        scope: Scope,
+        resources: ResourceFetcher,
+        *,
+        timeout_seconds: float,
+    ) -> RenderResult:
+        if (
+            page.rendered is not None
+            or not scope.permits(page.final_url)
+            or page.content_type != "text/html"
+            or not page.body
+        ):
+            raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+        if len(page.body) > self.config.max_input_bytes:
+            raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+        request = (
+            BrowserRequest(
+                config=self.config,
+                page=page,
+                user_agent=self._user_agent,
+                parent_network_namespace=os.readlink("/proc/self/ns/net"),
+            )
+            .model_dump_json()
+            .encode()
+            + b"\n"
+        )
+        if len(request) > self.config.max_protocol_bytes:
+            raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+        try:
+            async with (
+                asyncio.timeout(min(timeout_seconds, self.config.timeout_seconds)),
+                self._slots,
+            ):
+                self._check_artifacts()
+                return await self._run(request, page, scope, resources)
+        except TimeoutError:
+            raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED) from None
+        except (OSError, ValueError):
+            raise GhimeraRefused(RefusalCode.FETCH_FAILED) from None
+
+    async def _resource(
+        self,
+        request: ResourceRequest,
+        scope: Scope,
+        resources: ResourceFetcher,
+        retained: list[RenderResource],
+    ) -> RenderResource:
+        if (
+            request.method != "GET"
+            or request.resource_type not in self.config.resource_types
+            or not scope.permits(request.url)
+        ):
+            return RenderResource(
+                url=request.url,
+                method=request.method,
+                resource_type=request.resource_type,
+                refusal=RefusalCode.OUT_OF_SCOPE,
+                redirected_from=request.redirected_from,
+            )
+        try:
+            # The untrusted worker supplies an index, not authority. Reconstruct
+            # the chain from parent-retained responses before any new source I/O.
+            previous_index = request.redirected_from
+            urls = {request.url}
+            hops = 0
+            while previous_index is not None:
+                if previous_index >= len(retained):
+                    raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+                previous = retained[previous_index]
+                if (
+                    previous.refusal is not None
+                    or previous.status not in REDIRECT_STATUSES
+                    or previous.resource_type != request.resource_type
+                    or (
+                        hops == 0 and redirect_target(previous.url, previous.headers) != request.url
+                    )
+                ):
+                    raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+                hops += 1
+                if hops > self._redirect_limit or previous.url in urls:
+                    raise GhimeraRefused(RefusalCode.FETCH_FAILED)
+                urls.add(previous.url)
+                previous_index = previous.redirected_from
+            page = await resources.fetch(request.url)
+            if not scope.permits(page.final_url):
+                raise GhimeraRefused(RefusalCode.OUT_OF_SCOPE)
+            if page.final_url != request.url:
+                raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+            if page.status in REDIRECT_STATUSES:
+                target = redirect_target(page.final_url, page.headers)
+                if target is None:
+                    raise GhimeraRefused(RefusalCode.FETCH_FAILED)
+                if not scope.permits(target):
+                    raise GhimeraRefused(RefusalCode.OUT_OF_SCOPE)
+            elif page.status >= 300:
+                raise GhimeraRefused(RefusalCode.FETCH_FAILED)
+            elif page.content_type not in self.config.resource_content_types:
+                raise GhimeraRefused(RefusalCode.CONTENT_TYPE_UNWANTED)
+            if len(page.body) > self.config.max_input_bytes:
+                raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+            return RenderResource(
+                url=request.url,
+                method=request.method,
+                resource_type=request.resource_type,
+                final_url=page.final_url,
+                status=page.status,
+                content_type=page.content_type,
+                body=page.body,
+                source_sha256=hashlib.sha256(page.body).hexdigest(),
+                transport=page.transport,
+                source_session=page.source_session,
+                headers=page.headers,
+                redirected_from=request.redirected_from,
+            )
+        except GhimeraRefused as exc:
+            return RenderResource(
+                url=request.url,
+                method=request.method,
+                resource_type=request.resource_type,
+                refusal=exc.code,
+                redirected_from=request.redirected_from,
+            )
+
+    async def _run(
+        self,
+        request: bytes,
+        page: Page,
+        scope: Scope,
+        resources: ResourceFetcher,
+    ) -> RenderResult:
+        private_directory(self.config.work_directory)
+        scratch = Path(tempfile.mkdtemp(prefix="render-", dir=self.config.work_directory))
+        process: asyncio.subprocess.Process | None = None
+        starting: asyncio.Task[asyncio.subprocess.Process] | None = None
+        diagnostic: asyncio.Task[bytes] | None = None
+        retained: list[RenderResource] = []
+        total = len(request)
+        try:
+            # Browser AND driver inherit a new OS network namespace. No browser
+            # socket, DNS, service worker or WebRTC can reach the parent's network.
+            starting = asyncio.create_task(
+                asyncio.create_subprocess_exec(
+                    str(self.config.isolator),
+                    "--unshare-net",
+                    "--unshare-pid",
+                    "--die-with-parent",
+                    "--new-session",
+                    "--ro-bind",
+                    "/",
+                    "/",
+                    "--tmpfs",
+                    str(self.config.sandbox_work_directory.parent),
+                    "--bind",
+                    str(scratch),
+                    str(self.config.sandbox_work_directory),
+                    "--proc",
+                    "/proc",
+                    "--dev",
+                    "/dev",
+                    "--chdir",
+                    str(self.config.sandbox_work_directory),
+                    "--",
+                    sys.executable,
+                    "-m",
+                    "ghimera.browser_worker",
+                    stdin=asyncio.subprocess.PIPE,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    start_new_session=True,
+                    limit=self.config.max_protocol_bytes + 1,
+                    env={
+                        "PYTHONPATH": str(Path(__file__).resolve().parents[1]),
+                        "PYTHONNOUSERSITE": "1",
+                        "PYTHONDONTWRITEBYTECODE": "1",
+                        "TMPDIR": str(self.config.sandbox_work_directory),
+                        "XDG_CACHE_HOME": str(self.config.sandbox_work_directory / "cache"),
+                        "XDG_CONFIG_HOME": str(self.config.sandbox_work_directory / "config"),
+                    },
+                )
+            )
+            process = await asyncio.shield(starting)
+            if process.stdin is None or process.stdout is None:
+                raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+            diagnostic = asyncio.create_task(
+                read_bounded(
+                    process.stderr,
+                    self.config.max_diagnostic_bytes,
+                )
+            )
+            process.stdin.write(request)
+            await process.stdin.drain()
+            while True:
+                line = await process.stdout.readline()
+                total += len(line)
+                if total > self.config.max_protocol_bytes:
+                    raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+                if not line:
+                    raise GhimeraRefused(RefusalCode.FETCH_FAILED)
+                if diagnostic.done():
+                    diagnostic.result()  # Refuse overflow rather than logging vendor stderr.
+                try:
+                    requested = ResourceRequest.model_validate_json(line)
+                except ValueError:
+                    outcome = WorkerResult.model_validate_json(line)
+                    if outcome.refusal is not None:
+                        raise GhimeraRefused(outcome.refusal) from None
+                    result = outcome.result
+                    if result is None or (
+                        result.resources != tuple(retained)
+                        or result.source_url != page.final_url
+                        or result.source_sha256 != hashlib.sha256(page.body).hexdigest()
+                        or result.config_digest != self.config.content_digest()
+                        or result.browser_sha256 != self.config.executable_sha256
+                        or result.parent_network_namespace != os.readlink("/proc/self/ns/net")
+                        or len(result.html) > self.config.max_rendered_bytes
+                    ):
+                        raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT) from None
+                    try:
+                        result.validate_policy(self.config, max_redirects=self._redirect_limit)
+                    except ValueError:
+                        raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT) from None
+                    process.stdin.close()
+                    await diagnostic
+                    if await process.wait() != 0:
+                        raise GhimeraRefused(RefusalCode.FETCH_FAILED) from None
+                    return result
+                if requested.sequence != len(retained):
+                    raise GhimeraRefused(RefusalCode.ADAPTER_CONTRACT)
+                if len(retained) >= self.config.max_resources:
+                    raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+                fetched = await self._resource(requested, scope, resources, retained)
+                retained.append(fetched)
+                response = (
+                    ResourceResponse(
+                        sequence=requested.sequence,
+                        result=fetched,
+                    )
+                    .model_dump_json()
+                    .encode()
+                    + b"\n"
+                )
+                total += len(response)
+                if total > self.config.max_protocol_bytes:
+                    raise GhimeraRefused(RefusalCode.BUDGET_EXHAUSTED)
+                process.stdin.write(response)
+                await process.stdin.drain()
+        finally:
+            if process is None and starting is not None:
+                async with asyncio.timeout(self.config.cleanup_timeout_seconds):
+                    process = await asyncio.shield(starting)
+            if diagnostic is not None and not diagnostic.done():
+                diagnostic.cancel()
+            if diagnostic is not None:
+                await asyncio.gather(diagnostic, return_exceptions=True)
+            if process is not None:
+                if process.stdin is not None and not process.stdin.is_closing():
+                    process.stdin.transport.abort()
+                # Only this subprocess's fresh session is targeted. Descendants
+                # live in the child PID namespace, not shared browser sessions.
+                if process.returncode is None:
+                    try:
+                        os.killpg(process.pid, signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+
+                async def discard(stream: asyncio.StreamReader | None) -> None:
+                    if stream is not None:
+                        while await stream.read(65536):
+                            pass
+
+                async with asyncio.timeout(self.config.cleanup_timeout_seconds):
+                    await asyncio.gather(discard(process.stdout), discard(process.stderr))
+                    await process.wait()
+            shutil.rmtree(scratch)  # Exactly the owned mkdtemp directory, after reaping.

ghimera/browser_config.py

@@ -0,0 +1,69 @@
+"""Explicit, pinned browser execution policy; no inferred appliance paths."""
+
+import hashlib
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+Positive = Annotated[int, Field(strict=True, gt=0)]
+Seconds = Annotated[float, Field(gt=0, allow_inf_nan=False)]
+
+
+class BrowserConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, serialize_by_alias=True)
+    schema_version: Literal["chimera.browser/1"] = Field(alias="schema")
+    engine: Literal["patchright"]
+    executable: Path
+    executable_sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
+    isolator: Path
+    work_directory: Path
+    sandbox_work_directory: Path
+    max_workers: Positive
+    timeout_seconds: Seconds
+    cleanup_timeout_seconds: Seconds
+    max_input_bytes: Positive
+    max_rendered_bytes: Positive
+    max_protocol_bytes: Positive
+    max_diagnostic_bytes: Positive
+    max_resources: Positive
+    resource_types: tuple[
+        Literal["script", "stylesheet", "image", "font", "fetch", "xhr", "media"], ...
+    ]
+    resource_content_types: Annotated[tuple[str, ...], Field(min_length=1)]
+    ready_selector: Annotated[str, Field(min_length=1)] | None = None
+    settle_seconds: Annotated[float, Field(ge=0, allow_inf_nan=False)]
+    viewport_width: Positive
+    viewport_height: Positive
+
+    @model_validator(mode="after")
+    def coherent(self) -> "BrowserConfig":
+        if not all(
+            path.is_absolute()
+            for path in (
+                self.executable,
+                self.isolator,
+                self.work_directory,
+                self.sandbox_work_directory,
+            )
+        ):
+            raise ValueError("browser paths must be explicit absolute paths")
+        if (
+            self.sandbox_work_directory.parent != Path("/run")
+            or len(str(self.sandbox_work_directory).encode()) > 32
+        ):
+            raise ValueError("sandbox scratch must be a short immediate child of /run")
+        if len(set(self.resource_types)) != len(self.resource_types):
+            raise ValueError("resource types cannot repeat")
+        if any(not value or ";" in value for value in self.resource_content_types):
+            raise ValueError("resource MIME types must be normalized types without parameters")
+        if self.ready_selector is not None and not self.ready_selector.strip():
+            raise ValueError("ready selector must be nonblank")
+        if self.settle_seconds >= self.timeout_seconds:
+            raise ValueError("settle time must fit within the render deadline")
+        if self.max_protocol_bytes < 2 * max(self.max_input_bytes, self.max_rendered_bytes):
+            raise ValueError("protocol budget must hold encoded input and rendered output")
+        return self
+
+    def content_digest(self) -> str:
+        return hashlib.sha256(self.model_dump_json(by_alias=True).encode()).hexdigest()

ghimera/browser_types.py

@@ -0,0 +1,174 @@
+"""Source-bound rendering and resource evidence; DOM is not original source bytes."""
+
+import hashlib
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ghimera.browser_config import BrowserConfig
+from ghimera.refusals import RefusalCode
+from ghimera.response import REDIRECT_STATUSES, redirect_target
+from ghimera.source_session_types import SourceSessionUse
+from ghimera.transport_types import TransportEvidence
+
+Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
+Text = Annotated[str, Field(min_length=1)]
+
+
+class BrowserRecord(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+        frozen=True,
+        serialize_by_alias=True,
+        ser_json_bytes="base64",
+        val_json_bytes="base64",
+    )
+
+
+class ResourceRequest(BrowserRecord):
+    event: Literal["resource"] = "resource"
+    sequence: Annotated[int, Field(strict=True, ge=0)]
+    url: Text
+    method: Text
+    resource_type: Text
+    redirected_from: Annotated[int, Field(strict=True, ge=0)] | None = None
+
+
+class RenderResource(BrowserRecord):
+    url: Text
+    method: Text
+    resource_type: Text
+    final_url: str | None = None
+    status: Annotated[int, Field(strict=True, ge=100, le=399)] | None = None
+    content_type: str | None = None
+    body: bytes = b""
+    source_sha256: Digest | None = None
+    transport: TransportEvidence | None = None
+    headers: tuple[tuple[str, str], ...] = ()
+    refusal: RefusalCode | None = None
+    redirected_from: Annotated[int, Field(strict=True, ge=0)] | None = None
+    source_session: SourceSessionUse | None = None
+
+    @model_validator(mode="after")
+    def bound(self) -> "RenderResource":
+        if self.source_session is not None and self.source_session.request_url != self.final_url:
+            raise ValueError("source session selection must bind this resource URL")
+        if self.refusal is not None:
+            if (
+                any(
+                    value is not None
+                    for value in (
+                        self.status,
+                        self.source_sha256,
+                        self.content_type,
+                        self.final_url,
+                        self.transport,
+                        self.source_session,
+                    )
+                )
+                or self.body
+                or self.headers
+            ):
+                raise ValueError("refused resource cannot claim retained fetched content")
+        elif (
+            self.method != "GET"
+            or self.status is None
+            or not self.final_url
+            or not self.content_type
+            or self.source_sha256 != hashlib.sha256(self.body).hexdigest()
+        ):
+            raise ValueError("successful resource requires original content and digest")
+        elif self.status in REDIRECT_STATUSES and redirect_target(self.url, self.headers) is None:
+            raise ValueError("redirect resource requires one nonempty Location")
+        return self
+
+
+class ResourceResponse(BrowserRecord):
+    sequence: Annotated[int, Field(strict=True, ge=0)]
+    result: RenderResource
+
+
+class RenderResult(BrowserRecord):
+    schema_version: Literal["chimera.browser-render/1"] = Field(alias="schema")
+    source_url: Text
+    source_sha256: Digest
+    html: bytes
+    html_sha256: Digest
+    config_digest: Digest
+    browser_sha256: Digest
+    driver_revision: Literal["patchright@1.63.0"]
+    network_isolation: Literal["linux_network_namespace"]
+    parent_network_namespace: Annotated[str, Field(pattern=r"^net:\[\d+\]$")]
+    worker_network_namespace: Annotated[str, Field(pattern=r"^net:\[\d+\]$")]
+    resources: tuple[RenderResource, ...]
+
+    def validate_policy(self, policy: BrowserConfig, *, max_redirects: int | None = None) -> None:
+        if (
+            self.config_digest != policy.content_digest()
+            or self.browser_sha256 != policy.executable_sha256
+            or len(self.html) > policy.max_rendered_bytes
+            or len(self.resources) > policy.max_resources
+        ):
+            raise ValueError("rendering must bind its artifact, effective policy and limits")
+        if any(
+            item.refusal is None
+            and (
+                item.resource_type not in policy.resource_types
+                or (
+                    item.status not in REDIRECT_STATUSES
+                    and item.content_type not in policy.resource_content_types
+                )
+                or len(item.body) > policy.max_input_bytes
+                or item.url != item.final_url
+            )
+            for item in self.resources
+        ):
+            raise ValueError("successful rendering resources must obey the declared policy")
+        if max_redirects is not None:
+            for item in self.resources:
+                if item.refusal is not None:
+                    continue  # Over-cap refused hops are retained, but never fetched.
+                ancestor = item.redirected_from
+                hops = 0
+                urls = {item.url}
+                while ancestor is not None:
+                    previous = self.resources[ancestor]
+                    hops += 1
+                    if hops > max_redirects or previous.url in urls:
+                        raise ValueError("successful redirect resources exceed their HTTP policy")
+                    urls.add(previous.url)
+                    ancestor = previous.redirected_from
+
+    @model_validator(mode="after")
+    def bound(self) -> "RenderResult":
+        if not self.html or self.html_sha256 != hashlib.sha256(self.html).hexdigest():
+            raise ValueError("rendered DOM requires retained content and digest")
+        if self.parent_network_namespace == self.worker_network_namespace:
+            raise ValueError("browser worker must have its own network namespace")
+        for index, resource in enumerate(self.resources):
+            if resource.redirected_from is None:
+                continue
+            if resource.redirected_from >= index:
+                raise ValueError("redirect resource must follow a retained earlier resource")
+            previous = self.resources[resource.redirected_from]
+            if (
+                previous.refusal is not None
+                or previous.status not in REDIRECT_STATUSES
+                or redirect_target(previous.url, previous.headers) != resource.url
+                or previous.resource_type != resource.resource_type
+                or resource.method != "GET"
+            ):
+                raise ValueError("redirect chain must bind the actual retained Location and kind")
+        return self
+
+
+class WorkerResult(BrowserRecord):
+    event: Literal["result"] = "result"
+    result: RenderResult | None = None
+    refusal: RefusalCode | None = None
+
+    @model_validator(mode="after")
+    def outcome(self) -> "WorkerResult":
+        if (self.result is None) == (self.refusal is None):
+            raise ValueError("browser worker must report exactly one outcome")
+        return self