kiapi 0.1.0__py3-none-any.whl

kiapi/capabilities/web/_operations/resolve_searxng_config_dir.py

@@ -0,0 +1,21 @@
+import shutil
+from importlib.resources import files
+from pathlib import Path
+
+from kiapi.core.app import get_user_config_dir
+
+
+def resolve_searxng_config_dir() -> Path:
+    config_dir = Path(get_user_config_dir()).expanduser() / "searxng"
+    config_dir.mkdir(parents=True, exist_ok=True)
+
+    settings_path = config_dir / "settings.yml"
+    if not settings_path.exists():
+        bundled_settings = (
+            files("kiapi.capabilities.web.resources.searxng") / "settings.yml"
+        )
+        with bundled_settings.open("rb") as source:
+            with settings_path.open("wb") as target:
+                shutil.copyfileobj(source, target)
+
+    return config_dir

kiapi/capabilities/web/_services/__init__.py

@@ -0,0 +1 @@
+

kiapi/capabilities/web/_services/web_backend_process.py

@@ -0,0 +1,144 @@
+"""Foreground subprocess wrapper for web backend containers.
+
+The web capability treats SearXNG and Crawl4AI like resident subprocess models:
+``load`` starts a foreground ``docker run --rm`` process on a free localhost port,
+and ``release`` terminates that process. Docker is only an implementation detail
+of this payload.
+"""
+
+import socket
+import subprocess
+import time
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+from typing import BinaryIO, Self
+
+import httpx
+
+
+class WebBackendProcess:
+    def __init__(
+        self,
+        *,
+        name: str,
+        proc: subprocess.Popen,
+        base_url: str,
+        port: int,
+        log_file: BinaryIO,
+        log_path: Path,
+    ) -> None:
+        self.name = name
+        self.proc = proc
+        self.base_url = base_url
+        self.port = port
+        self.log_file = log_file
+        self.log_path = log_path
+        self.started_at = time.time()
+
+    def release(self) -> None:
+        proc = self.proc
+        if proc.poll() is None:
+            try:
+                proc.terminate()
+                proc.wait(timeout=15)
+            except Exception:
+                proc.kill()
+                proc.wait(timeout=5)
+        try:
+            self.log_file.close()
+        except Exception:
+            pass
+
+    @classmethod
+    def start(
+        cls,
+        *,
+        name: str,
+        image: str,
+        container_port: int,
+        log_dir: str,
+        ready_timeout_s: float,
+        healthcheck: Callable[[str], None],
+        extra_args: list[str] | None = None,
+        attempts: int = 3,
+    ) -> Self:
+        last_error: Exception | None = None
+        for _ in range(attempts):
+            port = _pick_free_port()
+            log_path = _log_path(log_dir, name, port)
+            log_file = log_path.open("ab")
+            container_name = f"kiapi-web-{name}-{uuid.uuid4().hex[:12]}"
+            cmd = [
+                "docker",
+                "run",
+                "--rm",
+                "--name",
+                container_name,
+                "-p",
+                f"127.0.0.1:{port}:{container_port}",
+                *(extra_args or []),
+                image,
+            ]
+            proc = subprocess.Popen(
+                cmd,
+                stdout=log_file,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+            )
+            backend = cls(
+                name=name,
+                proc=proc,
+                base_url=f"http://127.0.0.1:{port}",
+                port=port,
+                log_file=log_file,
+                log_path=log_path,
+            )
+            try:
+                _wait_ready(backend, ready_timeout_s, healthcheck)
+                return backend
+            except Exception as exc:
+                last_error = exc
+                backend.release()
+
+        assert last_error is not None
+        raise last_error
+
+
+def _pick_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
+def _log_path(log_dir: str, name: str, port: int) -> Path:
+    path = Path(log_dir).expanduser()
+    path.mkdir(parents=True, exist_ok=True)
+    return path / f"kiapi-web-{name}-{port}.log"
+
+
+def _wait_ready(
+    backend: WebBackendProcess,
+    timeout_s: float,
+    healthcheck: Callable[[str], None],
+) -> None:
+    deadline = time.monotonic() + timeout_s
+    last_error: Exception | None = None
+    while time.monotonic() < deadline:
+        if backend.proc.poll() is not None:
+            raise RuntimeError(
+                f"{backend.name} backend exited early "
+                f"(code {backend.proc.returncode}); log: {backend.log_path}"
+            )
+        try:
+            healthcheck(backend.base_url)
+            return
+        except (httpx.HTTPError, ValueError, RuntimeError) as exc:
+            last_error = exc
+            time.sleep(0.5)
+
+    detail = f": {last_error}" if last_error else ""
+    raise RuntimeError(
+        f"{backend.name} backend did not become ready within {timeout_s}s{detail}; "
+        f"log: {backend.log_path}"
+    )

kiapi/capabilities/web/_settings.py

@@ -0,0 +1,138 @@
+"""web capability defaults, read from ``KIAPI_WEB_`` env vars."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings_manager import SettingsManager
+
+
+class WebSettings(BaseSettings):
+    """Settings for Web search/fetch backends and request defaults."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="KIAPI_WEB_",
+        extra="ignore",
+        protected_namespaces=(),
+    )
+
+    # --------------------------------------------------
+    # search
+    # --------------------------------------------------
+
+    search_image: str = Field(
+        default="searxng/searxng:latest",
+        title="Search backend Docker image",
+        description="SearXNG Docker image name launched as the Web search backend.",
+    )
+
+    fetch_image: str = Field(
+        default="unclecode/crawl4ai:latest",
+        title="Fetch backend Docker image",
+        description="Crawl4AI Docker image name launched as the Web page fetch backend.",
+    )
+
+    backend_ready_timeout_s: float = Field(
+        default=60.0,
+        title="Backend ready timeout seconds",
+        description="Maximum seconds to wait for the search/fetch backend subprocess to become ready.",
+    )
+
+    backend_log_dir: str = Field(
+        default="/tmp/kiapi/logs/web",
+        title="Backend log directory",
+        description="Directory where search/fetch backend subprocess logs are written.",
+    )
+
+    timeout_s: float = Field(
+        default=10.0,
+        title="Search timeout seconds",
+        description="Maximum seconds allowed for one search request to SearXNG.",
+    )
+
+    default_categories: list[str] | None = Field(
+        default=None,
+        title="Default search categories",
+        description=(
+            "Categories passed to SearXNG when a request omits categories.\n"
+            "When None, SearXNG decides from its own defaults."
+        ),
+    )
+
+    default_engines: list[str] | None = Field(
+        default=None,
+        title="Default search engines",
+        description=(
+            "SearXNG engine names used when a request omits engines.\n"
+            "When None, SearXNG decides from its own defaults."
+        ),
+    )
+
+    default_language: str | None = Field(
+        default=None,
+        title="Default search language",
+        description=(
+            "Language code passed to SearXNG when a request omits language.\n"
+            "When None, SearXNG decides from its own defaults."
+        ),
+    )
+
+    default_safesearch: int | None = Field(
+        default=None,
+        title="Default SafeSearch level",
+        description=(
+            "SearXNG SafeSearch value used when a request omits safesearch.\n"
+            "Typically 0 disables it, 1 is moderate, and 2 is stricter. None "
+            "uses the SearXNG default."
+        ),
+    )
+
+    default_max_results: int | None = Field(
+        default=10,
+        title="Default maximum search results",
+        description=(
+            "Maximum number of results returned when a request omits max_results.\n"
+            "Set to None to disable client-side truncation in kiapi."
+        ),
+    )
+
+    # --------------------------------------------------
+    # fetch
+    # --------------------------------------------------
+
+    fetch_timeout_s: float = Field(
+        default=30.0,
+        title="Page fetch timeout seconds",
+        description="Maximum seconds allowed for one page fetch request to Crawl4AI.",
+    )
+
+    fetch_min_content_chars: int = Field(
+        default=1,
+        title="Minimum fetched content characters",
+        description=(
+            "Minimum stripped Markdown length before fetched content is treated as empty.\n"
+            "With 1, only truly empty renders fail. Higher values also reject "
+            "near-empty pages."
+        ),
+    )
+
+    fetch_filter: str = Field(
+        default="fit",
+        title="Fetch Markdown filter",
+        description=(
+            "Body extraction mode for Crawl4AI /md.\n"
+            "fit returns readability-pruned Markdown; raw returns Markdown "
+            "closer to the full DOM."
+        ),
+    )
+
+    fetch_cache: str = Field(
+        default="0",
+        title="Fetch cache-bust value",
+        description=(
+            "Cache control value passed to Crawl4AI /md.\n"
+            "0 lets the server cache normally. Changing the value can act as a "
+            "cache-busting identifier."
+        ),
+    )
+
+
+settings_manager = SettingsManager(WebSettings)

kiapi/capabilities/web/_views/fetch_params.py

@@ -0,0 +1,27 @@
+"""The complete contract for one Crawl4AI fetch call.
+
+Built from settings + request by ``resolve_fetch_params``: it carries the
+backend address, the Crawl4AI ``/md`` knobs (filter/cache), and the limits
+(timeout, concurrency, empty threshold) ``handle_fetch`` needs to issue the
+request and judge its result. Nothing here is caller-tunable in this op.
+"""
+
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class FetchParams(BaseModel):
+    url: str
+    format: Literal["markdown", "pdf"]
+
+    # Crawl4AI backend base URL.
+    base_url: str
+    timeout_s: float
+
+    # Crawl4AI /md knobs (ignored for the /pdf format).
+    filter: str
+    cache: str
+
+    # Empty-result threshold.
+    min_content_chars: int

kiapi/capabilities/web/_views/fetch_request.py

@@ -0,0 +1,29 @@
+"""web fetch request model.
+
+``GET /v1/web/fetch?url=...`` renders one page to readable text (or PDF) via the
+Crawl4AI backend. The output ``format`` is chosen by the request's ``Accept``
+header at the router and folded in here. Like ``search`` there is no ``mode``;
+the router runs it as a synchronous Job on the single worker.
+"""
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class FetchRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    # Absolute http(s) URL of the page to fetch. Passed to Crawl4AI verbatim.
+    url: str = Field(..., min_length=1)
+
+    # Resolved from Accept: text/markdown (default) → "markdown",
+    # application/pdf → "pdf".
+    format: Literal["markdown", "pdf"] = "markdown"
+
+    @field_validator("url")
+    @classmethod
+    def _require_http_scheme(cls, v: str) -> str:
+        if not v.startswith(("http://", "https://")):
+            raise ValueError("url must be an absolute http:// or https:// URL")
+        return v

kiapi/capabilities/web/_views/fetch_result.py

@@ -0,0 +1,20 @@
+"""web fetch result — the raw body the router streams back.
+
+fetch returns the rendered page directly (not a JSON envelope): ``content`` is
+the bytes to write, ``media_type`` the response Content-Type (``text/markdown``
+or ``application/pdf``). ``content_type`` is the *upstream* page's MIME (from the
+preflight probe), echoed in a response header for visibility. This mirrors the
+generation families' raw-bytes-by-default house style.
+"""
+
+from pydantic import BaseModel
+
+
+class FetchResult(BaseModel):
+    content: bytes
+    media_type: str
+
+    # Final URL after redirects, and the upstream page's detected MIME (or None
+    # when the probe could not determine it).
+    url: str
+    content_type: str | None

kiapi/capabilities/web/_views/search_params.py

@@ -0,0 +1,43 @@
+"""The complete contract for one SearXNG search call.
+
+Built from settings + request by ``resolve_search_params``: list fields are
+already comma-joined into SearXNG's wire form, and ``max_results`` carries the
+resolved truncation limit applied after the page is fetched. Everything needed
+to issue the HTTP request and shape the response lives here.
+"""
+
+from pydantic import BaseModel
+
+
+class SearchParams(BaseModel):
+    query: str
+
+    # SearXNG wire form: comma-joined strings, or None to omit the parameter.
+    categories: str | None
+    engines: str | None
+    language: str | None
+    time_range: str | None
+    safesearch: int | None
+    pageno: int
+
+    # Client-side truncation limit; None keeps the full page.
+    max_results: int | None
+
+    def query_params(self) -> dict:
+        """SearXNG ``/search`` querystring (``format=json`` is forced here)."""
+        params: dict[str, object] = {
+            "q": self.query,
+            "format": "json",
+            "pageno": self.pageno,
+        }
+        if self.categories is not None:
+            params["categories"] = self.categories
+        if self.engines is not None:
+            params["engines"] = self.engines
+        if self.language is not None:
+            params["language"] = self.language
+        if self.time_range is not None:
+            params["time_range"] = self.time_range
+        if self.safesearch is not None:
+            params["safesearch"] = self.safesearch
+        return params

kiapi/capabilities/web/_views/search_request.py

@@ -0,0 +1,93 @@
+"""web search request model.
+
+A thin wrapper over SearXNG's ``/search`` query parameters. ``query`` is passed
+through verbatim, so SearXNG's inline operators (``!images``, ``!wp``, ``:ja``,
+``site:``) work unchanged. ``format=json`` is added internally and is not a
+caller concern. There is no ``mode``: web search is run as a synchronous Job on
+the single worker.
+"""
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class SearchRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    query: str = Field(
+        ...,
+        min_length=1,
+        description=(
+            "Search query passed to SearXNG as-is. SearXNG inline operators "
+            "work here, for example `site:github.com kiapi`, `!wp Python`, "
+            "`!images cat`, or `:ja mlx`."
+        ),
+        examples=["site:github.com kiarina kiapi"],
+    )
+
+    categories: list[str] | None = Field(
+        default=None,
+        description=(
+            "SearXNG categories to search. Omit to use "
+            "`KIAPI_WEB_DEFAULT_CATEGORIES`; if that is unset, SearXNG chooses "
+            "its own default. Common values include `general`, `it`, `science`, "
+            "`news`, `images`, `videos`, `map`, `music`, `files`, and "
+            "`social_media`."
+        ),
+        examples=[["general"], ["it", "science"]],
+    )
+    engines: list[str] | None = Field(
+        default=None,
+        description=(
+            "Specific SearXNG engines to use. Omit to use "
+            "`KIAPI_WEB_DEFAULT_ENGINES`; if that is unset, SearXNG selects "
+            "engines from the requested categories. Examples: `google`, `bing`, "
+            "`duckduckgo`, `github`, `wikipedia`, `arxiv`."
+        ),
+        examples=[["duckduckgo"], ["github", "stackoverflow"]],
+    )
+
+    language: str | None = Field(
+        default=None,
+        description=(
+            "Optional SearXNG language code such as `ja`, `en`, or `en-US`. "
+            "Omit to use `KIAPI_WEB_DEFAULT_LANGUAGE`."
+        ),
+        examples=["ja", "en"],
+    )
+    time_range: Literal["day", "week", "month", "year"] | None = Field(
+        default=None,
+        description=(
+            "Optional freshness filter. Supported values are `day`, `week`, "
+            "`month`, and `year`. Support depends on the selected engines."
+        ),
+    )
+    safesearch: Literal[0, 1, 2] | None = Field(
+        default=None,
+        description=(
+            "SearXNG safe-search level: `0` disables filtering, `1` is moderate, "
+            "and `2` is strict. Omit to use `KIAPI_WEB_DEFAULT_SAFESEARCH`."
+        ),
+    )
+
+    page: int = Field(
+        default=1,
+        ge=1,
+        description=(
+            "One-based SearXNG result page (`pageno`). kiapi fetches exactly one "
+            "page per request; increase this value to request the next page."
+        ),
+    )
+
+    max_results: int | None = Field(
+        default=None,
+        ge=1,
+        description=(
+            "Maximum number of `results` to keep from the returned SearXNG page. "
+            "SearXNG has no native max-results parameter, so kiapi truncates "
+            "client-side. Omit to use `KIAPI_WEB_DEFAULT_MAX_RESULTS`; set the "
+            "server default to null to keep the full page."
+        ),
+        examples=[5, 10],
+    )

kiapi/capabilities/web/_views/search_response.py

@@ -0,0 +1,49 @@
+"""web search response.
+
+SearXNG's JSON is passed through with only the top level typed: the per-result
+dicts are forwarded verbatim (``list[dict]``) so SearXNG's evolving fields
+(``title``/``url``/``content``/``engine``/``score``/``img_src``/…) reach the
+caller without kiapi re-modelling them. ``results`` is truncated to the request's
+``max_results``; the auxiliary blocks (``answers``/``infoboxes``/``suggestions``)
+are forwarded untruncated because agents find direct answers there.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class SearchResponse(BaseModel):
+    query: str = Field(
+        description=(
+            "Normalized query returned by SearXNG. This usually matches the "
+            "submitted `query`, but SearXNG may normalize whitespace or operators."
+        )
+    )
+    results: list[dict[str, Any]] = Field(
+        description=(
+            "Search result dictionaries forwarded from SearXNG and truncated by "
+            "`max_results`. Common keys include `title`, `url`, `content`, "
+            "`engine`, `score`, `img_src`, and `publishedDate`; exact fields "
+            "depend on the engine and result type."
+        )
+    )
+    answers: list[Any] = Field(
+        description=("Direct-answer blocks from SearXNG, forwarded without truncation.")
+    )
+    infoboxes: list[Any] = Field(
+        description=(
+            "Knowledge-panel style infoboxes from SearXNG, forwarded without "
+            "truncation."
+        )
+    )
+    suggestions: list[Any] = Field(
+        description="Search suggestions from SearXNG, forwarded without truncation."
+    )
+    unresponsive_engines: list[Any] = Field(
+        description=(
+            "SearXNG `unresponsive_engines`: engines that failed or timed out for "
+            "this query, usually as `[engine, reason]` pairs. Results may still "
+            "be usable when this list is non-empty."
+        )
+    )

kiapi/capabilities/web/resources/searxng/settings.yml

@@ -0,0 +1,11 @@
+use_default_settings: true
+
+server:
+  # Local-only backend; override with KIAPI_SEARXNG_SECRET in the environment.
+  secret_key: "kiapi-searxng-local-dev-secret-change-me"
+
+search:
+  # kiapi consumes /search?format=json; SearXNG only enables json when listed here.
+  formats:
+    - html
+    - json