goodmem-nlweb 0.3.0__py3-none-any.whl

goodmem_nlweb/__init__.py

@@ -0,0 +1,40 @@
+"""GoodMem as an NLWeb retrieval provider.
+
+Registered by configuration -- NLWeb imports the class by path, so this
+package never has to be added to NLWeb's own source tree::
+
+    retrieval:
+      default:
+        import_path: goodmem_nlweb
+        class_name: GoodMemRetrievalProvider
+        base_url: https://localhost:8080
+        api_key: gm_…
+        space_name: nlweb
+
+NLWeb speaks Schema.org and filters by *site*; GoodMem stores text and
+metadata. ``goodmem_nlweb._schema`` documents the mapping.
+"""
+
+from goodmem_nlweb._connection import GoodMemConnection
+from goodmem_nlweb._schema import schema_to_text, to_memory_fields
+from goodmem_nlweb._spaces import GoodMemSpaceError
+from goodmem_nlweb.provider import (
+    GoodMemObjectLookupProvider,
+    GoodMemRetrievalProvider,
+    GoodMemUploadError,
+    upload_documents,
+)
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "GoodMemConnection",
+    "GoodMemObjectLookupProvider",
+    "GoodMemRetrievalProvider",
+    "GoodMemSpaceError",
+    "GoodMemUploadError",
+    "__version__",
+    "schema_to_text",
+    "to_memory_fields",
+    "upload_documents",
+]

goodmem_nlweb/_connection.py

@@ -0,0 +1,70 @@
+"""SDK client ownership.
+
+Ported from the AutoGen integration; the cancellation-token plumbing is
+removed because NLWeb has no equivalent.
+"""
+
+from __future__ import annotations
+
+from typing import cast
+
+from goodmem import AsyncGoodmem
+from typing_extensions import Self
+
+from goodmem_nlweb._typing import AsyncGoodmemClient
+
+
+class GoodMemConnection:
+    """Owns an ``AsyncGoodmem`` client, or borrows a caller-supplied one.
+
+    An injected client keeps its own server, credentials and TLS settings;
+    this class never closes it. Otherwise one client is created lazily and
+    closed by :meth:`close`. There is no process-wide client cache.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        verify_ssl: bool | str = True,
+        timeout: float = 60.0,
+        client: AsyncGoodmem | None = None,
+    ) -> None:
+        self._base_url = base_url
+        self._api_key = api_key
+        self._verify_ssl = verify_ssl
+        self._timeout = timeout
+        self._injected = client
+        self._owned: AsyncGoodmem | None = None
+
+    @property
+    def owns_client(self) -> bool:
+        return self._injected is None
+
+    def client(self) -> AsyncGoodmemClient:
+        if self._injected is not None:
+            return cast(AsyncGoodmemClient, self._injected)
+        if self._owned is None:
+            if not self._base_url or not self._api_key:
+                raise ValueError(
+                    "GoodMem base_url and api_key are required when no client is injected."
+                )
+            self._owned = AsyncGoodmem(
+                base_url=self._base_url.rstrip("/"),
+                api_key=self._api_key,
+                verify=self._verify_ssl,
+                timeout=self._timeout,
+            )
+        return cast(AsyncGoodmemClient, self._owned)
+
+    async def close(self) -> None:
+        if self._owned is not None:
+            await self._owned.close()
+            self._owned = None
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.close()

goodmem_nlweb/_ids.py

@@ -0,0 +1,56 @@
+"""The one check a GoodMem id passes before it can reach a URL path.
+
+The goodmem SDK builds request paths by interpolating ids unencoded
+(``f"/v1/spaces/{space_id}/memories"``, ``f"/v1/memories/{id}"``), and httpx
+resolves dot segments before it sends. A space id of ``../spaces/<other>``
+is therefore a request against a different space, and ``<id>#x`` or
+``<id>?x=1`` cuts the path short -- measured, not assumed; see
+``tests/test_ids.py``. Neither the client nor the server can be relied on to
+stop that, so it is stopped here, before any request is made.
+
+Every GoodMem id (space, memory, embedder, reranker, ...) is a UUID, so the
+rule is simple: a canonical UUID is accepted and lowercased, and anything
+else is refused.
+
+Where it is called:
+
+* configured ids are checked when the provider is built, so a bad option
+  fails when NLWeb loads it rather than on the first request;
+* an id that goes into a URL path is checked again immediately before the
+  SDK call that uses it, whatever its source -- configuration, or the
+  server's own answer.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+import uuid
+
+# fullmatch, not ^...$: in a Python regex `$` also matches before a trailing
+# newline, which would let "<uuid>\n" through.
+_CANONICAL_UUID = re.compile(
+    r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+)
+
+
+def require_uuid(value: Any, field: str) -> str:
+    """Return ``value`` as a lowercase canonical UUID, or raise ``ValueError``.
+
+    A ``uuid.UUID`` is accepted as-is. The looser spellings ``uuid.UUID()``
+    parses -- braces, ``urn:uuid:``, no hyphens -- are refused: an id that is
+    about to become a path segment has exactly one acceptable form.
+
+    The refused value is not repeated in the message. A mistyped option can
+    be a credential, and this message ends up in logs.
+    """
+    if isinstance(value, uuid.UUID):
+        return str(value)
+    if isinstance(value, str) and _CANONICAL_UUID.fullmatch(value):
+        return value.lower()
+    raise ValueError(
+        f"{field} must be a UUID (8-4-4-4-12 hexadecimal digits). GoodMem ids "
+        "are UUIDs, and any other value is refused before a request is made "
+        "because the SDK places ids in the URL path unencoded, where a value "
+        "such as '../' would send the request to a different endpoint."
+    )

goodmem_nlweb/_results.py

@@ -0,0 +1,110 @@
+"""Retrieval event handling: statuses, chunk/memory joining, scores."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+from goodmem.models.good_mem_status import GoodMemStatus
+from goodmem.models.retrieve_memory_event import RetrieveMemoryEvent
+
+# Notices that carry no loss of results.
+#
+# FEATURE_DISABLED is informational unconditionally. The server defines it as
+# "feature disabled due to missing configuration" (common.proto, under
+# "Informational status messages (non-error)"): the caller did not configure
+# an optional feature, so nothing the caller asked for is missing. A feature
+# that was requested and could not be delivered arrives as a different code
+# (NOT_FOUND, RERANKING_FAILED, ...). Retrieval status contract, Q1.
+_INFORMATIONAL_CODES = frozenset({"LLM_CAPABILITY_INFERRED", "FEATURE_DISABLED"})
+
+
+def is_informational(status: GoodMemStatus) -> bool:
+    """True for notices that do not indicate incomplete retrieval.
+
+    An unrecognized code is deliberately not informational: the SDK decodes
+    codes it does not know as ``None``, and a status from a newer server must
+    be surfaced rather than assumed harmless.
+    """
+    return status.code is not None and status.code in _INFORMATIONAL_CODES
+
+
+def classify(events: Sequence[RetrieveMemoryEvent]) -> tuple[list[dict[str, Any]], bool]:
+    """Split statuses into what to report and whether results are incomplete.
+
+    Known informational notices are dropped. Known failures mark the result
+    degraded. Codes this SDK does not recognize are surfaced as ``UNKNOWN``
+    and mark the result degraded, but never discard chunks and never raise:
+    a newer server must not be able to break retrieval.
+    """
+    surfaced: list[dict[str, Any]] = []
+    degraded = False
+    for event in events:
+        status = event.status
+        if status is None or is_informational(status):
+            continue
+        entry = status.model_dump(exclude_none=True)
+        if status.code is None:
+            entry["code"] = "UNKNOWN"
+            entry["unrecognized"] = True
+        surfaced.append(entry)
+        degraded = True
+    return surfaced, degraded
+
+
+def hits_from_events(
+    events: Iterable[RetrieveMemoryEvent], *, reranked: bool
+) -> list[dict[str, Any]]:
+    """Join chunks to their memory definitions by UUID, ignoring event order.
+
+    Deduplicates by ``chunk_id``: two chunks of one memory are two distinct
+    results, and collapsing them by ``memory_id`` would drop matching content.
+
+    Server ordering and raw scores are preserved. ``score_kind`` records where
+    a score came from, because a reranker score and a vector score are not on
+    the same scale and must not be compared or thresholded together.
+    """
+    events = list(events)
+    memories = {
+        event.memory_definition.memory_id: event.memory_definition
+        for event in events
+        if event.memory_definition is not None
+    }
+
+    hits: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for event in events:
+        item = event.retrieved_item
+        if item is None or item.chunk is None:
+            continue
+        reference = item.chunk
+        chunk = reference.chunk
+        if chunk is None or not chunk.chunk_text:
+            continue
+        if chunk.chunk_id in seen:
+            continue
+        seen.add(chunk.chunk_id)
+
+        memory = memories.get(chunk.memory_id) or item.memory
+        metadata: dict[str, Any] = dict(getattr(memory, "metadata", None) or {})
+        hits.append(
+            {
+                "chunk_text": chunk.chunk_text,
+                "chunk_id": chunk.chunk_id,
+                "memory_id": chunk.memory_id,
+                "space_id": getattr(memory, "space_id", None),
+                "source": getattr(memory, "original_content_ref", None) or chunk.memory_id,
+                "score": reference.relevance_score,
+                "score_kind": "reranker" if reranked else "vector",
+                # The memory's own metadata, which is what a caller stored.
+                "metadata": metadata,
+            }
+        )
+    return hits
+
+
+def abstract_reply(events: Iterable[RetrieveMemoryEvent]) -> dict[str, Any] | None:
+    for event in events:
+        if event.abstract_reply is not None:
+            return event.abstract_reply.model_dump(exclude_none=True)
+    return None

goodmem_nlweb/_schema.py

@@ -0,0 +1,135 @@
+"""Mapping between Schema.org objects and GoodMem memories.
+
+NLWeb speaks Schema.org: every retrieved item is ``(url, raw_schema_object,
+site)``. GoodMem stores text and metadata. The mapping is:
+
+===================  ==========================================
+NLWeb                GoodMem
+===================  ==========================================
+``url``              ``metadata["url"]`` -- also the identity
+``site``             ``metadata["site"]`` -- what NLWeb filters on
+``raw_schema_object````metadata["schema_json"]`` (a JSON string)
+(text to embed)      the memory's ``original_content``
+===================  ==========================================
+
+The embedded text is *not* the raw JSON. Embedding a JSON blob buries the
+words a query would match under punctuation and key names, so the text is
+extracted from the fields a reader would actually read. The untouched object
+is kept alongside, because NLWeb returns it verbatim to the caller.
+
+This mirrors how NLWeb's own Qdrant provider stores things: a payload with
+``url``, ``schema_json``, ``name`` and ``site``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+#: Schema.org fields worth embedding, in the order a reader would meet them.
+_TEXT_FIELDS = (
+    "name",
+    "headline",
+    "alternateName",
+    "description",
+    "abstract",
+    "text",
+    "articleBody",
+)
+
+MAX_EMBED_CHARS = 8000
+
+
+def schema_to_text(schema: Any) -> str:
+    """Extract the human-readable text from a Schema.org object.
+
+    Falls back to a compact JSON rendering only when no known text field is
+    present, because retrieving nothing is worse than embedding a blob.
+    """
+    if isinstance(schema, str):
+        try:
+            schema = json.loads(schema)
+        except ValueError:
+            return schema[:MAX_EMBED_CHARS]
+    if isinstance(schema, list):
+        return "\n\n".join(schema_to_text(item) for item in schema)[:MAX_EMBED_CHARS]
+    if not isinstance(schema, dict):
+        return str(schema)[:MAX_EMBED_CHARS]
+
+    parts: list[str] = []
+    for field in _TEXT_FIELDS:
+        value = schema.get(field)
+        if isinstance(value, str) and value.strip():
+            parts.append(value.strip())
+    if not parts:
+        return json.dumps(schema, ensure_ascii=False)[:MAX_EMBED_CHARS]
+    return "\n\n".join(parts)[:MAX_EMBED_CHARS]
+
+
+def schema_name(schema: Any) -> str:
+    """The display name NLWeb shows for an item."""
+    if isinstance(schema, str):
+        try:
+            schema = json.loads(schema)
+        except ValueError:
+            return ""
+    if isinstance(schema, list):
+        return schema_name(schema[0]) if schema else ""
+    if not isinstance(schema, dict):
+        return ""
+    for field in ("name", "headline", "alternateName"):
+        value = schema.get(field)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return ""
+
+
+def schema_url(schema: Any, fallback: str = "") -> str:
+    """The URL NLWeb uses as an item's identity."""
+    if isinstance(schema, str):
+        try:
+            schema = json.loads(schema)
+        except ValueError:
+            return fallback
+    if isinstance(schema, list):
+        return schema_url(schema[0], fallback) if schema else fallback
+    if not isinstance(schema, dict):
+        return fallback
+    for field in ("url", "@id", "identifier"):
+        value = schema.get(field)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return fallback
+
+
+def to_memory_fields(
+    document: Any, *, site: str = "", url: str = ""
+) -> tuple[str, dict[str, Any]]:
+    """Return ``(text_to_embed, metadata)`` for one Schema.org document.
+
+    ``document`` may be the object itself, a JSON string, or an NLWeb-style
+    wrapper carrying ``url`` / ``site`` / ``schema_json`` keys.
+    """
+    schema: Any = document
+    if isinstance(document, dict) and (
+        "schema_json" in document or "raw_schema_object" in document
+    ):
+        schema = document.get("schema_json") or document.get("raw_schema_object")
+        site = document.get("site") or site
+        url = document.get("url") or url
+
+    resolved_url = url or schema_url(schema)
+    if not resolved_url:
+        raise ValueError(
+            "Every NLWeb document needs a url: it is the item's identity and "
+            "the key NLWeb deduplicates on. Pass url= or include one in the "
+            "Schema.org object."
+        )
+
+    raw = schema if isinstance(schema, str) else json.dumps(schema, ensure_ascii=False)
+    return schema_to_text(schema), {
+        "url": resolved_url,
+        "site": site,
+        "name": schema_name(schema),
+        "schema_json": raw,
+    }

goodmem_nlweb/_spaces.py

@@ -0,0 +1,124 @@
+"""Resolving a space by name.
+
+Attaching by name is idempotent reuse: an existing space with the requested
+embedder is returned as-is, and one built on a different embedder is an error
+rather than a silent mismatch. Retrieval against the wrong embedder returns
+plausible-looking nonsense, so it must never happen quietly.
+
+This matches the ActivePieces connector's behaviour. GoodMem does not enforce
+unique space names, so an ambiguous name is also an error.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from goodmem_nlweb._typing import AsyncGoodmemClient
+
+# A name is not unique, so the lookup is bounded rather than unbounded.
+MAX_CANDIDATES = 200
+
+
+class GoodMemSpaceError(RuntimeError):
+    """A space could not be resolved unambiguously."""
+
+
+async def find_by_name(client: AsyncGoodmemClient, name: str) -> list[Any]:
+    """Return every space named exactly ``name``.
+
+    ``name_filter`` is a server-side substring match, so each candidate is
+    re-checked for an exact name. Iterating the SDK's ``Page`` follows
+    pagination for us; ``max_items`` bounds it.
+    """
+    page = await client.spaces.list(name_filter=name, max_items=MAX_CANDIDATES)
+    return [s async for s in page if getattr(s, "name", None) == name]
+
+
+def embedder_ids(space: Any) -> list[str]:
+    return [
+        e.embedder_id
+        for e in (getattr(space, "space_embedders", None) or [])
+        if getattr(e, "embedder_id", None)
+    ]
+
+
+async def resolve(
+    client: AsyncGoodmemClient,
+    *,
+    name: str,
+    embedder_id: str | None = None,
+    create: bool = False,
+    chunking_config: dict[str, Any] | None = None,
+) -> str:
+    """Return the id of the space called ``name``.
+
+    With ``create=True`` a missing space is created. An existing space is
+    reused only when ``embedder_id`` is unset or already configured on it.
+    """
+    matches = await find_by_name(client, name)
+    if len(matches) > 1:
+        raise GoodMemSpaceError(
+            f"{len(matches)} spaces are named {name!r}. Pass space_id to choose "
+            "one; GoodMem does not require space names to be unique."
+        )
+
+    if matches:
+        space = matches[0]
+        configured = embedder_ids(space)
+        if embedder_id and embedder_id not in configured:
+            raise GoodMemSpaceError(
+                f"Space {name!r} ({space.space_id}) uses embedder(s) "
+                f"{', '.join(configured) or 'none'}, not {embedder_id}. "
+                "Retrieval across mismatched embedders returns meaningless "
+                "results. Pick the existing embedder or use another name."
+            )
+        return str(space.space_id)
+
+    if not create:
+        raise GoodMemSpaceError(
+            f"No space named {name!r}. Pass space_id, or set create_space=True "
+            "to create it."
+        )
+    if not embedder_id:
+        raise GoodMemSpaceError(
+            f"Cannot create space {name!r} without embedder_id. List the "
+            "available embedders with the GoodMem SDK or the goodmem CLI."
+        )
+
+    request: dict[str, Any] = {
+        "name": name,
+        "space_embedders": [
+            {"embedderId": embedder_id, "defaultRetrievalWeight": 1.0}
+        ],
+        # The server rejects a create without one.
+        "default_chunking_config": chunking_config or _DEFAULT_CHUNKING,
+    }
+    try:
+        space = await client.spaces.create(**request)
+    except Exception as exc:
+        # Another process may have created it between the lookup and the
+        # create. Re-resolve rather than fail the caller.
+        if not _is_conflict(exc):
+            raise
+        again = await find_by_name(client, name)
+        if len(again) == 1:
+            return str(again[0].space_id)
+        raise
+    return str(space.space_id)
+
+
+_DEFAULT_CHUNKING: dict[str, Any] = {
+    "recursive": {
+        "chunkSize": 256,
+        "chunkOverlap": 25,
+        "separators": ["\n\n", "\n", ". ", " ", ""],
+        "keepStrategy": "KEEP_END",
+        "separatorIsRegex": False,
+        "lengthMeasurement": "CHARACTER_COUNT",
+    }
+}
+
+
+def _is_conflict(exc: Exception) -> bool:
+    status = getattr(exc, "status_code", None) or getattr(exc, "status", None)
+    return status == 409 or "conflict" in type(exc).__name__.lower()

goodmem_nlweb/_typing.py

@@ -0,0 +1,49 @@
+"""Typing for the SDK client.
+
+The SDK attaches its API groups (``spaces``, ``memories``, …) to the client
+dynamically, so they carry no annotations and a type checker cannot see them.
+These Protocols describe only the surface this package uses. They add no
+runtime layer: the objects passed around are the SDK's own.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Protocol
+
+
+class _SpacesAPI(Protocol):
+    async def list(self, **kwargs: Any) -> Any: ...
+    async def get(self, *, id: str) -> Any: ...
+    async def create(self, **kwargs: Any) -> Any: ...
+    async def update(self, *, id: str, request: Any) -> Any: ...
+    async def delete(self, *, id: str) -> None: ...
+
+
+class _MemoriesAPI(Protocol):
+    async def list(self, **kwargs: Any) -> Any: ...
+    async def get(self, **kwargs: Any) -> Any: ...
+    async def create(self, **kwargs: Any) -> Any: ...
+    async def delete(self, *, id: str) -> None: ...
+    async def retrieve(self, **kwargs: Any) -> Any: ...
+    # Sequence, not list: `list` above shadows the builtin in this scope.
+    async def batch_create(self, *, requests: Sequence[Any]) -> Any: ...
+
+
+class _ModelsAPI(Protocol):
+    async def list(self, **kwargs: Any) -> Any: ...
+
+
+class AsyncGoodmemClient(Protocol):
+    """The parts of ``goodmem.AsyncGoodmem`` this package calls."""
+
+    @property
+    def spaces(self) -> _SpacesAPI: ...
+    @property
+    def memories(self) -> _MemoriesAPI: ...
+    @property
+    def embedders(self) -> _ModelsAPI: ...
+    @property
+    def rerankers(self) -> _ModelsAPI: ...
+
+    async def close(self) -> None: ...