python-corekit 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

corekit/html/state.py

@@ -0,0 +1,56 @@
+import asyncio
+from typing import Any
+
+from corekit.utils import safe_list
+
+
+class ScraperState:
+    # Sentinel value to signal the end of the stream
+    FINISHED_TOKEN = object()
+
+    _seen: set[str]
+    _queue: asyncio.Queue[str]
+
+    def __init__(self, start_urls: list[str] | None = None) -> None:
+        self._seen = set()
+        self._queue = asyncio.Queue()
+        self._output_queue: asyncio.Queue[Any] = asyncio.Queue()
+        self._lock: asyncio.Lock = asyncio.Lock()
+
+        for url in safe_list(start_urls):
+            self.enqueue_no_wait(url)
+
+    def enqueue_no_wait(self, url: str) -> None:
+        if url in self._seen:
+            return
+        self._seen.add(url)
+        self._queue.put_nowait(url)
+
+    async def enqueue(self, url: str) -> None:
+        async with self._lock:
+            if url in self._seen:
+                return
+            self._seen.add(url)
+            await self._queue.put(url)
+
+    async def get_next_url(self) -> str:
+        return await self._queue.get()
+
+    def task_done(self) -> None:
+        self._queue.task_done()
+
+    async def record_output(self, data: Any) -> None:
+        await self._output_queue.put(data)
+
+    async def get_output(self) -> Any:
+        return await self._output_queue.get()
+
+    def output_task_done(self) -> None:
+        self._output_queue.task_done()
+
+    def is_empty(self) -> bool:
+        return self._queue.empty()
+
+    async def join(self) -> None:
+        await self._queue.join()
+        await self.record_output(self.FINISHED_TOKEN)

corekit/http/__init__.py

@@ -25,7 +25,7 @@ from corekit.http.exceptions import (
     URLMismatchError,
 )
 from corekit.http.exponential_backoff import ExponentialBackoff
-from corekit.http.response import BaseApiResponse
+from corekit.http.response import BaseApiResponse, BaseHttpResponse
 from corekit.http.status import HTTPStatusCode
 from corekit.http.stream import SseDecoder, SseJsonDecoder, SseTextDecoder, StreamDecoder
 
@@ -35,6 +35,7 @@ __all__ = [
     "BaseApiClient",
     "BaseApiResponse",
     "BaseHttpClient",
+    "BaseHttpResponse",
     "ConflictErrorException",
     "ExponentialBackoff",
     "ForbiddenException",

corekit/http/api.py

@@ -1,13 +1,29 @@
 """
-API clients: a ``BaseHttpClient`` that always talks to one origin.
+API clients: a ``BaseHttpClient`` that always talks to one origin and parses JSON.
 """
 
+from http import HTTPMethod
 from typing import Any
 
+import httpx
+
+from corekit.concurrency.decorators import allow_sync
 from corekit.http.client import BaseHttpClient
+from corekit.http.response import BaseApiResponse
+from corekit.http.status import HTTPStatusCode
 
 __all__ = ["BaseApiClient"]
 
+_API_METHODS = frozenset(
+    {
+        HTTPMethod.GET,
+        HTTPMethod.POST,
+        HTTPMethod.PUT,
+        HTTPMethod.PATCH,
+        HTTPMethod.DELETE,
+    }
+)
+
 
 class BaseApiClient(BaseHttpClient):
     """
@@ -16,9 +32,61 @@ class BaseApiClient(BaseHttpClient):
     Strict by default, so an absolute URL that does not share ``base_url`` is
     a bug rather than a request to somewhere else. ``base_url`` must be
     available during ``__init__`` — a property on the subclass is the usual way.
+
+    Supports the usual mutating verbs. Responses are ``BaseApiResponse`` with
+    structured ``data`` when the body is JSON.
     """
 
-    def __init__(self, *args: Any, strict: bool = True, **kwargs: Any) -> None:
-        super().__init__(*args, strict=strict, **kwargs)
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        kwargs.setdefault("strict", True)
+        super().__init__(*args, **kwargs)
         if not self.base_url:
             raise ValueError("BaseApiClient requires a non-empty base_url")
+
+    @property
+    def supported_methods(self) -> frozenset[HTTPMethod]:
+        """
+        API clients may send the common REST verbs.
+        """
+        return _API_METHODS
+
+    def _to_response(self, raw: httpx.Response) -> BaseApiResponse:
+        """
+        Normalize a httpx response into a ``BaseApiResponse`` with parsed ``data``.
+        """
+        try:
+            loaded: Any = raw.json()
+        except ValueError:
+            loaded = None
+
+        return BaseApiResponse(
+            status_code=HTTPStatusCode.coerce(raw.status_code),
+            text=raw.text,
+            data=BaseApiResponse.parse_data(loaded) if loaded is not None else None,
+            headers=dict(raw.headers),
+            cookies=dict(raw.cookies),
+            url=str(raw.url),
+        )
+
+    async def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await super().request(method, url, **kwargs)  # type: ignore[return-value]
+
+    @allow_sync
+    async def get(self, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await self.request(HTTPMethod.GET, url, **kwargs)
+
+    @allow_sync
+    async def post(self, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await self.request(HTTPMethod.POST, url, **kwargs)
+
+    @allow_sync
+    async def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await self.request(HTTPMethod.PUT, url, **kwargs)
+
+    @allow_sync
+    async def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await self.request(HTTPMethod.PATCH, url, **kwargs)
+
+    @allow_sync
+    async def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
+        return await self.request(HTTPMethod.DELETE, url, **kwargs)

corekit/http/client.py

@@ -22,10 +22,10 @@ absolute URL should be a bug.
 
     client = GithubClient()
     response = client.get("/user")          # or await client.get("/user")
-    response.data["login"]
+    response.data.login
 
-Every response comes back as a ``BaseApiResponse``, so callers see one shape
-regardless of what the endpoint returned.
+``BaseHttpClient`` returns a ``BaseHttpResponse`` (text + metadata).
+``BaseApiClient`` adds structured JSON on ``data``.
 """
 
 from collections.abc import AsyncIterator
@@ -39,9 +39,10 @@ import httpx
 from corekit.concurrency.decorators import allow_sync
 from corekit.http.exceptions import RETRYABLE_STATUS_CODES, UnsupportedMethodError, URLMismatchError
 from corekit.http.exponential_backoff import ExponentialBackoff
-from corekit.http.response import BaseApiResponse
+from corekit.http.response import BaseHttpResponse
 from corekit.http.status import HTTPStatusCode
 from corekit.observability.benchmarkable import Benchmarkable
+from corekit.utils import safe_dict
 
 __all__ = ["BaseHttpClient"]
 
@@ -54,27 +55,22 @@ def _without_query(url: str) -> str:
     return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
 
 
-_DEFAULT_METHODS = frozenset(
-    {
-        HTTPMethod.GET,
-        HTTPMethod.POST,
-        HTTPMethod.PUT,
-        HTTPMethod.PATCH,
-        HTTPMethod.DELETE,
-    }
-)
+_DEFAULT_METHODS = frozenset({HTTPMethod.GET})
 
 
 class BaseHttpClient(Benchmarkable):
     """
     Base for HTTP clients. Override ``base_url`` and, usually, ``headers``.
 
-    ``strict=False`` accepts a foreign absolute URL as-is. ``BaseApiClient``
-    turns that into an error.
+    GET-only by default — fetching is the transport's job. Subclass
+    ``BaseApiClient`` (or override ``supported_methods``) when mutating verbs
+    are required. ``strict=False`` accepts a foreign absolute URL as-is;
+    ``BaseApiClient`` turns that into an error.
     """
 
     def __init__(
         self,
+        base_url: str = "",
         retries: int = 3,
         timeout: float = 30.0,
         strict: bool = False,
@@ -82,6 +78,7 @@ class BaseHttpClient(Benchmarkable):
         http_client: httpx.AsyncClient | None = None,
     ) -> None:
         super().__init__()
+        self._base_url = base_url
         self.retries = retries
         self.timeout = timeout
         self.strict = strict
@@ -93,7 +90,7 @@ class BaseHttpClient(Benchmarkable):
     @property
     def supported_methods(self) -> frozenset[HTTPMethod]:
         """
-        HTTP methods this client will send. Override to narrow or extend the set.
+        HTTP methods this client will send. Defaults to GET only.
         """
         return _DEFAULT_METHODS
 
@@ -109,7 +106,7 @@ class BaseHttpClient(Benchmarkable):
         """
         The root every relative path is resolved against.
         """
-        return ""
+        return self._base_url
 
     def _build_url(self, url: str) -> str:
         """
@@ -124,30 +121,21 @@ class BaseHttpClient(Benchmarkable):
             return url
         return urljoin(self.base_url, url)
 
-    @staticmethod
-    def _to_response(raw: httpx.Response) -> BaseApiResponse:
+    def _to_response(self, raw: httpx.Response) -> BaseHttpResponse:
         """
-        Normalize a httpx response into a BaseApiResponse.
+        Normalize a httpx response into a ``BaseHttpResponse``.
 
-        The body is decoded as JSON when it parses, and left as text otherwise,
-        so a caller never has to guard against a non-JSON error page.
+        Text and metadata only — JSON parsing belongs on ``BaseApiClient``.
         """
-        try:
-            data = raw.json()
-        except ValueError:
-            data = {}
-        if not isinstance(data, dict):
-            data = {"data": data}
-
-        return BaseApiResponse(
+        return BaseHttpResponse(
             status_code=HTTPStatusCode.coerce(raw.status_code),
             text=raw.text,
-            data=data,
             headers=dict(raw.headers),
             cookies=dict(raw.cookies),
+            url=str(raw.url),
         )
 
-    async def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseApiResponse:
+    async def request(self, method: HTTPMethod, url: str, **kwargs: Any) -> BaseHttpResponse:
         """
         Send a request asynchronously, retrying retryable statuses.
         """
@@ -156,9 +144,7 @@ class BaseHttpClient(Benchmarkable):
 
         target = self._build_url(url)
         headers = dict(self.headers)
-        extra_headers = kwargs.pop("headers", None)
-        if extra_headers:
-            headers.update(extra_headers)
+        headers.update(safe_dict(kwargs.pop("headers", {})))
         # Fresh each request: ExponentialBackoff keeps its own attempt count.
         backoff = ExponentialBackoff(retries=self.retries, base_delay=self.base_delay)
 
@@ -176,7 +162,7 @@ class BaseHttpClient(Benchmarkable):
         headers: dict[str, str],
         backoff: ExponentialBackoff,
         kwargs: dict[str, Any],
-    ) -> BaseApiResponse:
+    ) -> BaseHttpResponse:
         while True:
             raw = await client.request(str(method), target, headers=headers, **kwargs)
             if raw.status_code not in RETRYABLE_STATUS_CODES or backoff.did_timeout():
@@ -191,7 +177,7 @@ class BaseHttpClient(Benchmarkable):
         Open a streaming response and yield the raw ``httpx.Response``.
 
         Holds the connection for the life of the ``async with`` block. Unlike
-        ``request``, the body is not buffered into a ``BaseApiResponse`` — the
+        ``request``, the body is not buffered into a response model — the
         caller reads it (for example with ``aiter_lines`` for SSE). Retries are
         not applied: a mid-stream failure cannot be replayed safely.
         """
@@ -214,21 +200,5 @@ class BaseHttpClient(Benchmarkable):
                 yield response
 
     @allow_sync
-    async def get(self, url: str, **kwargs: Any) -> BaseApiResponse:
+    async def get(self, url: str, **kwargs: Any) -> BaseHttpResponse:
         return await self.request(HTTPMethod.GET, url, **kwargs)
-
-    @allow_sync
-    async def post(self, url: str, **kwargs: Any) -> BaseApiResponse:
-        return await self.request(HTTPMethod.POST, url, **kwargs)
-
-    @allow_sync
-    async def put(self, url: str, **kwargs: Any) -> BaseApiResponse:
-        return await self.request(HTTPMethod.PUT, url, **kwargs)
-
-    @allow_sync
-    async def patch(self, url: str, **kwargs: Any) -> BaseApiResponse:
-        return await self.request(HTTPMethod.PATCH, url, **kwargs)
-
-    @allow_sync
-    async def delete(self, url: str, **kwargs: Any) -> BaseApiResponse:
-        return await self.request(HTTPMethod.DELETE, url, **kwargs)

corekit/http/response.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+
 from pydantic import BaseModel, Field
 
 from corekit.exceptions import CoreHTTPException
@@ -14,16 +18,23 @@ from corekit.http.exceptions import (
     UnprocessableEntityException,
 )
 from corekit.http.status import HTTPStatusCode
-from corekit.schemas.pydantic.fields import DefaultDictField
+from corekit.schemas.models.arbitrary import ArbitraryModel
+from corekit.schemas.pydantic.fields import DefaultDictField, DefaultEmptyField
 from corekit.schemas.types import AnyDict
 
 
-class BaseApiResponse(BaseModel):
+class BaseHttpResponse(BaseModel):
+    """
+    Transport-level HTTP response: status, decoded body text, and metadata.
+
+    No JSON parsing happens here — subclasses add structured ``data`` when needed.
+    """
+
     status_code: HTTPStatusCode | int = Field(..., description="HTTP status code")
-    text: str = Field(..., description="Response text")
-    data: AnyDict = DefaultDictField()
+    text: str = Field(..., description="Decoded response body")
     headers: AnyDict = DefaultDictField()
     cookies: AnyDict = DefaultDictField()
+    url: str = Field(default="", description="Final response URL after redirects")
 
     @property
     def ok(self) -> bool:
@@ -32,7 +43,7 @@ class BaseApiResponse(BaseModel):
         """
         return int(self.status_code) < 400
 
-    def raise_for_status(self) -> "BaseApiResponse":
+    def raise_for_status(self) -> BaseHttpResponse:
         """
         Raise a typed HTTP exception for a failure status.
 
@@ -59,3 +70,25 @@ class BaseApiResponse(BaseModel):
         if exception_type is None:
             raise CoreHTTPException(status_code=int(self.status_code), detail=detail)
         raise exception_type(detail=detail)
+
+
+class BaseApiResponse(BaseHttpResponse):
+    """
+    API response with structured ``data`` parsed from JSON when present.
+
+    ``data`` keeps the wire shape: objects become ``ArbitraryModel``, arrays stay
+    lists, primitives stay primitives, and non-JSON bodies yield ``None``.
+    """
+
+    data: Any = DefaultEmptyField(description="Structured JSON body, if any")
+
+    def raise_for_status(self) -> BaseApiResponse:
+        super().raise_for_status()
+        return self
+
+    @staticmethod
+    def parse_data(raw_json: Any) -> Any:
+        """
+        Coerce a decoded JSON value into structured ``data``.
+        """
+        return ArbitraryModel.coerce(raw_json)

corekit/llm/client.py

@@ -15,6 +15,7 @@ from typing import Any, Literal, overload
 from corekit.concurrency.decorators import allow_sync
 from corekit.http import BaseApiClient, BaseApiResponse, SseJsonDecoder
 from corekit.llm.messages import ChatTurn, Message
+from corekit.schemas import ArbitraryModel
 
 __all__ = ["ChatCompletionsClient", "DEFAULT_LLM_TIMEOUT"]
 
@@ -162,9 +163,9 @@ class ChatCompletionsClient(BaseApiClient):
         )
         response = await self.post("chat/completions", json=body)
         response.raise_for_status()
-        if not response.data:
+        if not isinstance(response.data, ArbitraryModel):
             raise ValueError(f"chat/completions returned non-JSON body: {response.text!r}")
-        return dict(response.data)
+        return response.data.model_dump()
 
     async def _complete_stream(self, body: dict[str, Any]) -> AsyncIterator[dict[str, Any]]:
         self.info(

corekit/llm/tools/loop.py

@@ -24,7 +24,7 @@ from corekit.llm.tools.detection import MAX_TOOL_ITERATIONS, ToolCallLoopDetecto
 from corekit.llm.tools.registry import ToolRegistry
 from corekit.llm.wire import ChoiceDelta, CompletionTurn, UsageInfo
 from corekit.observability import Benchmarkable
-from corekit.utils.coercion import safe_dict
+from corekit.utils import parse_dict
 
 __all__ = ["ToolLoop"]
 
@@ -224,12 +224,15 @@ class ToolLoop(Benchmarkable):
     async def _execute_ui_tool(self, call: ToolCall) -> UIComponentEvent:
         result = await self._registry.aexecute(call.name, call.id, call.arguments)
         self.info(f"[ToolLoop] UI result — is_error: {result.is_error}")
+        parsed = parse_dict(result.content, default={})
         if result.is_error:
             self.warning(f"[ToolLoop] UI tool failed: {result.content}")
+        elif not parsed.success:
+            self.warning(f"[ToolLoop] UI tool {call.name!r} returned non-mapping component: {result.content!r}")
         return UIComponentEvent(
             tool_call_id=call.id,
             name=call.name,
-            component=safe_dict(result.content),
+            component=parsed.value,
         )
 
     def _record_metrics(

corekit/llm/wire.py

@@ -13,7 +13,7 @@ from pydantic import BaseModel, Field
 from corekit.llm.enum import WireField
 from corekit.llm.events import UsageEvent
 from corekit.llm.tools.base import ToolCall
-from corekit.utils import safe_dict, safe_string
+from corekit.utils import parse_dict, safe_dict, safe_string
 from corekit.utils.collections import attr_or_key
 
 __all__ = [
@@ -89,12 +89,18 @@ class ToolCallAccumulator(BaseModel):
         """
         Parse accumulated argument JSON into a ``ToolCall``.
         """
-        parsed: dict[str, Any] = {}
         try:
-            parsed = json.loads(self.arguments)
+            loaded: Any = json.loads(self.arguments)
         except json.JSONDecodeError:
-            logger.warning("Bad tool args for %r: %r", self.name, self.arguments)
-        return ToolCall(id=self.id, name=self.name, arguments=safe_dict(parsed))
+            logger.warning(f"Bad tool args for {self.name!r}: {self.arguments!r}")
+            return ToolCall(id=self.id, name=self.name, arguments={})
+
+        parsed = parse_dict(loaded, default={})
+        if not parsed.success:
+            logger.warning(
+                f"Tool args for {self.name!r} decoded to non-mapping {type(loaded).__name__}: {self.arguments!r}",
+            )
+        return ToolCall(id=self.id, name=self.name, arguments=parsed.value)
 
 
 class ChoiceDelta(BaseModel):

corekit/schemas/__init__.py

@@ -6,6 +6,13 @@ raises where it is introduced rather than somewhere further along.
 """
 
 from corekit.schemas.enum import IntegerEnum, StringEnum, ValidatingEnum
+from corekit.schemas.models.arbitrary import ArbitraryModel
 from corekit.schemas.version import SemanticVersion
 
-__all__ = ["IntegerEnum", "SemanticVersion", "StringEnum", "ValidatingEnum"]
+__all__ = [
+    "ArbitraryModel",
+    "IntegerEnum",
+    "SemanticVersion",
+    "StringEnum",
+    "ValidatingEnum",
+]

corekit/schemas/models/arbitrary.py

@@ -1,11 +1,55 @@
-from pydantic import BaseModel
+"""
+Recursive attribute access over unstructured JSON-shaped data.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+from pydantic import BaseModel, ConfigDict, model_validator
 
 
 class ArbitraryModel(BaseModel):
     """
-    Base model for arbitrary data. This model is used to take unstructured data and convert it into
-    a structured format that can be easily mapped to a model. This model is capable of parsing any type of data
-    and converting it into a structured format that can be easily accessed and manipulated.
+    Accept any object payload and expose nested dicts as models.
+
+    Lists keep their shape; dict elements inside them are coerced the same way.
+    ``model_dump()`` returns plain nested dicts; ``model["key"]`` works for
+    callers that still treat the payload like a mapping.
     """
 
-    pass
+    model_config = ConfigDict(extra="allow")
+
+    @classmethod
+    def coerce(cls, value: Any) -> Any:
+        """
+        Recursively turn dicts into ``ArbitraryModel`` instances.
+        """
+        if isinstance(value, cls):
+            return value
+        if isinstance(value, Mapping) and not isinstance(value, (str, bytes)):
+            return cls.model_validate(dict(value))
+        if isinstance(value, list):
+            return [cls.coerce(item) for item in value]
+        return value
+
+    @model_validator(mode="after")
+    def _coerce_extras(self) -> ArbitraryModel:
+        extras = self.__pydantic_extra__
+        if not extras:
+            return self
+        for key, value in list(extras.items()):
+            extras[key] = type(self).coerce(value)
+        return self
+
+    def __bool__(self) -> bool:
+        return bool(self.model_dump())
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return getattr(self, key)
+        except AttributeError as exc:
+            raise KeyError(key) from exc
+
+    def __contains__(self, key: object) -> bool:
+        return isinstance(key, str) and key in self.model_dump()

corekit/utils/__init__.py

@@ -1,15 +1,34 @@
 """
 Small, dependency-free helpers.
 
-Four groups: total conversions that never raise (``safe_*``), string assembly
-for human-readable output, identifier generation, and nested-dictionary lookup.
-Plus timezone-aware time helpers and the placeholder callbacks (``void``,
-``raise_exc``, ``true_validator``, ``false_validator``) that callbacks-by-
-configuration need.
+Five groups: flexible parsers with caller-controlled fallbacks (``parse_*``),
+total conversions that never raise (``safe_*`` wrappers around those parsers),
+string assembly for human-readable output, identifier generation, and
+nested-dictionary lookup. Plus timezone-aware time helpers and the placeholder
+callbacks (``void``, ``raise_exc``, ``true_validator``, ``false_validator``)
+that callbacks-by-configuration need.
 """
 
-from corekit.utils.coercion import safe_dict, safe_float, safe_int, safe_list, safe_string, safe_tuple
-from corekit.utils.collections import UNSET, MultiMatch, attr_or_key, keygetter, repeated_get, split_list
+from corekit.utils.coercion import (
+    safe_dict,
+    safe_float,
+    safe_int,
+    safe_list,
+    safe_set,
+    safe_string,
+    safe_tuple,
+)
+from corekit.utils.collections import (
+    UNSET,
+    MultiMatch,
+    aiterator,
+    attr_or_key,
+    iterator,
+    keygetter,
+    kv_string,
+    repeated_get,
+    split_list,
+)
 from corekit.utils.ids import (
     SHORTCODE_CHARS,
     generate_session_token,
@@ -17,6 +36,16 @@ from corekit.utils.ids import (
     generate_shortcodes,
     generate_uuid,
 )
+from corekit.utils.parsers import (
+    ParsedValue,
+    parse_dict,
+    parse_float,
+    parse_int,
+    parse_list,
+    parse_set,
+    parse_string,
+    parse_tuple,
+)
 from corekit.utils.payload import Payload, decode_payload, encode_payload
 from corekit.utils.raise_exc import raise_exc
 from corekit.utils.text import join_lines, list_to_english, sanitize_filename, truncate
@@ -28,7 +57,9 @@ __all__ = [
     "SHORTCODE_CHARS",
     "UNSET",
     "MultiMatch",
+    "ParsedValue",
     "Payload",
+    "aiterator",
     "attr_or_key",
     "decode_payload",
     "encode_payload",
@@ -38,16 +69,26 @@ __all__ = [
     "generate_shortcodes",
     "generate_uuid",
     "isoformat_now",
+    "iterator",
     "join_lines",
     "keygetter",
+    "kv_string",
     "list_to_english",
+    "parse_dict",
     "parse_docker_timestamp",
+    "parse_float",
+    "parse_int",
+    "parse_list",
+    "parse_set",
+    "parse_string",
+    "parse_tuple",
     "raise_exc",
     "repeated_get",
     "safe_dict",
     "safe_float",
     "safe_int",
     "safe_list",
+    "safe_set",
     "safe_string",
     "safe_tuple",
     "sanitize_filename",