sphinx_learn 0.1.0__py3-none-any.whl

sphinx_ai/__init__.py

@@ -0,0 +1,17 @@
+"""Public interface for the sphinx_ai package."""
+
+from ._ingestion import _Ingestion as Ingestion
+from ._file import _File as File
+from ._knowledge_base import (
+    _Document as Document,
+    _KnowledgeBaseClient as KnowledgeBaseClient,
+    _createKnowledgeBaseClient as createKnowledgeBaseClient,
+)
+
+__all__ = [
+    "createKnowledgeBaseClient",
+    "Document",
+    "File",
+    "Ingestion",
+    "KnowledgeBaseClient",
+]

sphinx_ai/_environment.py

@@ -0,0 +1,6 @@
+"""Environment-specific configuration for the Sphinx client."""
+
+import os
+
+_SPHINX_API_URL = os.getenv("SPHINX_API_URL", "https://api.prod.sphinx.ai")
+_SPHINX_UI_URL = os.getenv("SPHINX_UI_URL", "https://app.sphinx.ai")

sphinx_ai/_file.py

@@ -0,0 +1,193 @@
+"""Validated in-memory files for knowledge-base ingestion."""
+
+import json
+import re
+from dataclasses import dataclass
+
+
+_MIN_SOURCE_PATH_CHARACTERS = 1
+_MAX_SOURCE_PATH_CHARACTERS = 512
+
+# Keep aligned with common/src/contracts/ingestContracts.json.
+_BYTES_PER_MIB = 1024**2
+_MAX_TEXT_FILE_BYTES = 4 * _BYTES_PER_MIB
+_MAX_BINARY_DOCUMENT_BYTES = 100 * _BYTES_PER_MIB
+_MAX_DOCUMENT_ESTIMATED_TOKENS = 250_000
+_BINARY_DOCUMENT_EXTENSIONS = frozenset(
+    {
+        ".pdf",
+        ".docx",
+        ".doc",
+        ".pptx",
+        ".ppt",
+        ".png",
+        ".jpg",
+        ".jpeg",
+        ".xlsx",
+        ".ods",
+    }
+)
+_JSON_LINES_EXTENSIONS = frozenset({".jsonl", ".ndjson"})
+_STRUCTURED_TEXT_EXTENSIONS = _JSON_LINES_EXTENSIONS | {".json", ".ipynb"}
+
+# Match Thoth's text decoding and token estimate.
+_TEXT_ENCODING = "utf-8-sig"
+_CHARACTERS_PER_ESTIMATED_TOKEN = 4
+
+
+@dataclass(frozen=True)
+class _File:
+    """An immutable in-memory document validated on creation.
+
+    Args:
+        path: A string of 1–512 characters. Allowed characters are ASCII
+            letters, digits, ! - _ . * ' ( ), and / as a path separator.
+            Empty, '.' and '..' path segments are forbidden.
+        content: Raw document bytes, subject to the constraints below.
+
+    Content constraints (all size limits are inclusive):
+        - Supported binary documents: at most 100 MiB (104,857,600 bytes).
+          Extensions are .pdf, .doc/.docx, .ppt/.pptx, .png, .jpg/.jpeg,
+          .xlsx and .ods, matched case-insensitively. Binary document
+          internals are not parsed or validated by this constructor.
+        - All other files: at most 4 MiB (4,194,304 bytes), valid UTF-8,
+          and no null bytes. A leading UTF-8 BOM is allowed and excluded
+          from the decoded character count.
+        - Text: at most 1,000,000 decoded characters, equivalent to the
+          250,000-token estimate computed as ceil(character_count / 4).
+        - .json: must parse as JSON.
+        - .jsonl/.ndjson: each nonblank line must parse as JSON.
+        - .ipynb: must parse as a JSON object containing a 'cells' list;
+          every cell must be an object.
+        - JSON parsing rejects NaN, Infinity and -Infinity.
+
+
+    Raises:
+        TypeError: path is not a string or content is not bytes.
+        ValueError: The path or content violates a constraint above.
+    """
+
+    path: str
+    content: bytes
+
+    def __post_init__(self) -> None:
+        _checkPathIsGood(self)
+        _checkContentIsGood(self)
+
+
+def _checkPathIsGood(file: _File) -> None:
+    if not isinstance(file.path, str):
+        raise TypeError(f"File.path must be a string; got {type(file.path).__name__}")
+    if not _MIN_SOURCE_PATH_CHARACTERS <= len(file.path) <= _MAX_SOURCE_PATH_CHARACTERS:
+        raise ValueError(
+            f"File.path must contain {_MIN_SOURCE_PATH_CHARACTERS} to "
+            f"{_MAX_SOURCE_PATH_CHARACTERS} characters; got {len(file.path):,} in {file.path!r}"
+        )
+    if re.fullmatch(r"[a-zA-Z0-9!_.*'()/\-]+", file.path) is None:
+        unsupported_characters = sorted(set(re.findall(r"[^a-zA-Z0-9!_.*'()/\-]", file.path)))
+        raise ValueError(
+            "File.path must use only S3-safe characters: "
+            "letters A-Z/a-z, digits, ! - _ . * ' ( ), and / as a path separator; "
+            f"got path {file.path!r} with unsupported characters "
+            f"{unsupported_characters!r}"
+        )
+    if any(part in ("", ".", "..") for part in file.path.split("/")):
+        raise ValueError(f"File.path must not contain empty, '.' or '..' segments; got {file.path!r}")
+
+
+def _checkContentIsGood(file: _File) -> None:
+    if not isinstance(file.content, bytes):
+        raise TypeError(f"File.content must be bytes; got {type(file.content).__name__}")
+
+    basename = file.path.rsplit("/", 1)[-1]
+    stem, separator, suffix = basename.rpartition(".")
+    extension = f".{suffix.lower()}" if separator and stem else ""
+    is_binary = extension in _BINARY_DOCUMENT_EXTENSIONS
+
+    if is_binary:
+        _checkBinaryFileIsGood(file)
+    else:
+        _checkTextFileIsGood(file, extension)
+
+
+def _checkBinaryFileIsGood(file: _File) -> None:
+    if len(file.content) > _MAX_BINARY_DOCUMENT_BYTES:
+        raise ValueError(
+            f"File ({file.path!r}) must be at most {_MAX_BINARY_DOCUMENT_BYTES / _BYTES_PER_MIB:g} MiB "
+            f"({_MAX_BINARY_DOCUMENT_BYTES:,} bytes); got {len(file.content):,} bytes"
+        )
+
+
+def _checkTextFileIsGood(file: _File, extension: str) -> None:
+    _checkTextFileIsAtMost4MiB(file)
+    text = _tryParseTextAsUTF(file, extension)
+    _checkTextIsAtMost250kEstimatedTokens(text, file.path)
+    if extension == ".ipynb":
+        _checkIpynbIsValid(text, file.path)
+    elif extension in _STRUCTURED_TEXT_EXTENSIONS:
+        _checkJsonIsValid(text, file.path, extension)
+
+
+def _checkTextFileIsAtMost4MiB(file: _File) -> None:
+    if len(file.content) > _MAX_TEXT_FILE_BYTES:
+        raise ValueError(
+            f"File ({file.path!r}) must be at most {_MAX_TEXT_FILE_BYTES / _BYTES_PER_MIB:g} MiB "
+            f"({_MAX_TEXT_FILE_BYTES:,} bytes); got {len(file.content):,} bytes"
+        )
+
+
+def _tryParseTextAsUTF(file: _File, extension: str) -> str:
+    """Return decoded UTF-8 text without a leading BOM, or raise ValueError."""
+    # The caller checks the byte limit before allocating the decoded string.
+    try:
+        if b"\x00" in file.content:
+            raise ValueError(f"text contains a null byte at byte offset {file.content.index(0):,}")
+        return file.content.decode(_TEXT_ENCODING)
+    except ValueError as error:
+        raise ValueError(
+            f"File ({file.path!r}) must be a supported binary document "
+            f"({', '.join(sorted(_BINARY_DOCUMENT_EXTENSIONS))}) or UTF-8 text without null bytes; "
+            f"got extension {extension!r}, {len(file.content):,} bytes: {error}"
+        ) from error
+
+
+def _checkTextIsAtMost250kEstimatedTokens(text: str, path: str) -> None:
+    decoded_characters = len(text)
+    estimated_tokens = (decoded_characters + _CHARACTERS_PER_ESTIMATED_TOKEN - 1) // _CHARACTERS_PER_ESTIMATED_TOKEN
+    if estimated_tokens > _MAX_DOCUMENT_ESTIMATED_TOKENS:
+        raise ValueError(
+            f"File ({path!r}) exceeds the {_MAX_DOCUMENT_ESTIMATED_TOKENS:,} estimated-token limit; "
+            f"got {estimated_tokens:,} estimated tokens ({decoded_characters:,} decoded characters)"
+        )
+
+
+def _checkJsonIsValid(text: str, path: str, extension: str) -> None:
+    try:
+        if extension in _JSON_LINES_EXTENSIONS:
+            for line_number, line in enumerate(text.split("\n"), start=1):
+                if not line.strip(" \t\r"):
+                    continue
+                try:
+                    json.loads(line, parse_constant=_rejectNonFiniteJsonConstant)
+                except ValueError as error:
+                    raise ValueError(f"invalid JSON on line {line_number}: {error}") from error
+        else:
+            json.loads(text, parse_constant=_rejectNonFiniteJsonConstant)
+    except ValueError as error:
+        raise ValueError(f"File ({path!r}) contains invalid {extension} content: {error}") from error
+
+
+def _checkIpynbIsValid(text: str, path: str) -> None:
+    try:
+        parsed = json.loads(text, parse_constant=_rejectNonFiniteJsonConstant)
+        if not isinstance(parsed, dict) or not isinstance(parsed.get("cells"), list):
+            raise ValueError("notebook must be an object with a cells list")
+        if any(not isinstance(cell, dict) for cell in parsed["cells"]):
+            raise ValueError("notebook cells must be objects")
+    except ValueError as error:
+        raise ValueError(f"File ({path!r}) contains invalid .ipynb content: {error}") from error
+
+
+def _rejectNonFiniteJsonConstant(value: str) -> None:
+    # Match Thoth: Python's JSON decoder otherwise accepts NaN and Infinity.
+    raise ValueError(f"Invalid JSON constant: {value}")

sphinx_ai/_ingestion.py

@@ -0,0 +1,12 @@
+"""Scaffolding for knowledge-base ingestion jobs."""
+
+
+class _Ingestion:
+    """Handle returned by a knowledge-base ingestion request."""
+
+    def __init__(self, url: str) -> None:
+        self._url = url
+
+    def job_url(self) -> str:
+        """Return the URL for this ingestion job."""
+        return self._url

sphinx_ai/_knowledge_base.py

@@ -0,0 +1,206 @@
+"""Client helpers for creating and populating Sphinx knowledge bases."""
+
+import base64
+import hashlib
+import json
+import mimetypes
+from typing import TypeAlias
+
+import urllib3
+
+from .gen.object_storage_pb import CreatePresignedPutRequest
+from .gen.ra.jobs_connect import RaJobServiceClientSync
+from .gen.ra.jobs_pb import SubmitJobRequest
+from .gen.sessions.compute_pb import ComputeEnvironmentConfiguration, ComputeEnvironmentType
+from .gen.sphinx_connect import SphinxRuntimeServiceClientSync
+from ._environment import _SPHINX_API_URL, _SPHINX_UI_URL
+from ._ingestion import _Ingestion
+from ._file import _File
+
+
+# SDK input policies.
+_MIN_DOCUMENT_COUNT = 1
+_MAX_DOCUMENT_COUNT = 20_000
+_MAX_PROMPT_CHARACTERS = 10_000
+_BYTES_PER_GIB = 1024**3
+_MAX_COLLECTION_BYTES = 2_684_354_560
+
+_HTTP = urllib3.PoolManager()
+
+
+_Document: TypeAlias = _File
+
+
+class _KnowledgeBaseClient:
+    """Client for fitting a Sphinx knowledge base to your data"""
+
+    def __init__(self, api_key: str, project_id: str) -> None:
+        self.api_key = api_key
+        self.project_id = project_id
+        self._runtime_client = SphinxRuntimeServiceClientSync(_SPHINX_API_URL)
+        self._ra_jobs_client = RaJobServiceClientSync(_SPHINX_API_URL)
+
+    def fit(
+        self,
+        prompt: str,
+        docs: list[_Document],
+    ) -> _Ingestion:
+        """Fits the Sphinx Knowledge base to your documents
+        by creating a new ingest job
+
+        Args:
+            prompt: Instructions that guide how the documents are ingested.
+            docs: Documents to upload and ingest.
+
+        Returns:
+            A handle containing the URL of the newly created ingestion job.
+
+        Raises:
+            TypeError: The prompt or documents have the wrong type.
+            ValueError: Prompt or batch limits documented by _checkInputIsGood are exceeded.
+        """
+        _checkInputIsGood(prompt, docs)
+        refs = [self._upload_document(doc) for doc in docs]
+        id = self._submit_job(prompt, docs, refs)
+
+        return _Ingestion(f"{_SPHINX_UI_URL}/knowledge-base/projects/{self.project_id}/ingest/{id}")
+
+    def _upload_document(self, doc: _Document) -> str:
+        content_type = self._guess_mimetype(doc.path)
+        presigned = self._runtime_client.create_presigned_put(
+            CreatePresignedPutRequest(
+                project_id=self.project_id,
+                content_type=content_type,
+                filename=doc.path,
+            ),
+            headers=self._auth_headers(),
+        )
+
+        upload_response = _HTTP.request(
+            "PUT",
+            presigned.presigned_url,
+            body=doc.content,
+            headers=dict(presigned.required_headers),
+        )
+
+        if not 200 <= upload_response.status < 300:
+            raise RuntimeError(f'Failed to upload "{doc.path}": {upload_response.status} {upload_response.reason}')
+
+        return presigned.ref
+
+    def _submit_job(
+        self,
+        prompt: str,
+        doc: list[_Document],
+        refs: list[str],
+    ) -> str:
+        # Omitting target domains lets the server scope the job to the root domain.
+        res = self._ra_jobs_client.submit_job(
+            SubmitJobRequest(
+                job_type="agentic_ingest",
+                project_id=self.project_id,
+                args=json.dumps(
+                    {
+                        "prompt": prompt.strip(),
+                        "files": [
+                            {
+                                "ref": ref,
+                                "filename": doc.path,
+                                "content_type": self._guess_mimetype(doc.path),
+                                "size_bytes": len(doc.content),
+                                "sha256": hashlib.sha256(doc.content).hexdigest(),
+                            }
+                            for doc, ref in zip(doc, refs, strict=True)
+                        ],
+                    }
+                ),
+                compute_environment_configuration=ComputeEnvironmentConfiguration(
+                    name="Sphinx Cloud",
+                    type=ComputeEnvironmentType.SPHINX_CLOUD,
+                ),
+            ),
+            headers=self._auth_headers(),
+        )
+
+        return res.job.id
+
+    def _auth_headers(self) -> dict[str, str]:
+        """Build the HTTP headers used to authenticate Sphinx API requests."""
+        credentials = f":{self.api_key}".encode("utf-8")
+        encoded_api_key = base64.b64encode(credentials).decode("ascii")
+        return {
+            "Authorization": f"Basic {encoded_api_key}",
+            "X-Sphinx-Version": "1.0.0",
+        }
+
+    def _guess_mimetype(self, path: str) -> str:
+        """Infer a MIME type from a path, defaulting to binary content."""
+        return mimetypes.guess_type(path)[0] or "application/octet-stream"
+
+
+def _createKnowledgeBaseClient(api_key: str, project_id: str) -> _KnowledgeBaseClient:
+    """Create a client for a Sphinx knowledge base.
+
+    Args:
+        api_key: PAT Token key used to authenticate requests to Sphinx. The corresponding
+            user must be an admin.
+        project_id: Identifier of the Sphinx project that owns the knowledge base.
+
+    Returns:
+        A configured knowledge-base client.
+    """
+    return _KnowledgeBaseClient(api_key, project_id)
+
+
+def _checkInputIsGood(prompt: str, docs: list[_Document]) -> None:
+    """Validate prompt and batch policies before uploads.
+
+    Requires 1–20,000 validated File objects, at most 2.5 GiB in total, and
+    a nonblank prompt of at most 10,000 characters. File construction owns
+    path and content validation. Wrong types raise TypeError; invalid values
+    raise ValueError.
+    """
+
+    def checkInputTypesAreCorrect() -> None:
+        if not isinstance(prompt, str):
+            raise TypeError(f"prompt must be a string; got {type(prompt).__name__}")
+        if not isinstance(docs, list):
+            raise TypeError(f"docs must be a list of File objects; got {type(docs).__name__}")
+        for index, doc in enumerate(docs):
+            if not isinstance(doc, _File):
+                raise TypeError(f"docs[{index}] must be a File; got {type(doc).__name__}")
+
+    def checkDocumentCountIsAtMost20k() -> None:
+        if len(docs) > _MAX_DOCUMENT_COUNT:
+            raise ValueError(f"docs must contain at most {_MAX_DOCUMENT_COUNT:,} documents; got {len(docs):,}")
+
+    def checkAtLeastOneDocument() -> None:
+        if len(docs) < _MIN_DOCUMENT_COUNT:
+            raise ValueError(f"docs must contain at least {_MIN_DOCUMENT_COUNT} document; got {len(docs)}")
+
+    def checkPromptSizeAtMost10kCharacters() -> None:
+        if len(prompt) > _MAX_PROMPT_CHARACTERS:
+            raise ValueError(f"prompt must contain at most {_MAX_PROMPT_CHARACTERS:,} characters; got {len(prompt):,}")
+
+    def checkPromptIsNotEmpty() -> None:
+        if not prompt.strip():
+            raise ValueError(
+                "prompt must not be empty or contain only whitespace; "
+                f"got {len(prompt):,} characters, 0 after stripping whitespace"
+            )
+
+    def checkTotalDocSizeIsAtMost2p5Gb() -> None:
+        total_bytes = sum(len(doc.content) for doc in docs)
+        if total_bytes > _MAX_COLLECTION_BYTES:
+            raise ValueError(
+                f"docs must total at most {_MAX_COLLECTION_BYTES / _BYTES_PER_GIB:g} GiB "
+                f"({_MAX_COLLECTION_BYTES:,} bytes); "
+                f"got {total_bytes:,} bytes across {len(docs):,} documents"
+            )
+
+    checkInputTypesAreCorrect()
+    checkPromptSizeAtMost10kCharacters()
+    checkPromptIsNotEmpty()
+    checkDocumentCountIsAtMost20k()
+    checkAtLeastOneDocument()
+    checkTotalDocSizeIsAtMost2p5Gb()

sphinx_ai/gen/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

sphinx_ai/gen/object_storage_pb.py

@@ -0,0 +1,115 @@
+# Generated from object_storage.proto. DO NOT EDIT.
+# Generated by protoc-gen-py v0.3.0 with parameter "".
+# ruff: noqa: PGH004
+# ruff: noqa
+# fmt: off
+
+from __future__ import annotations
+
+from typing import Literal, TYPE_CHECKING, TypeAlias
+
+from protobuf import Message
+from protobuf._codegen import file_desc
+
+if TYPE_CHECKING:
+    from protobuf import DescFile
+
+
+_CreatePresignedPutRequestFields: TypeAlias = Literal["project_id", "content_type", "filename"]
+
+class CreatePresignedPutRequest(Message[_CreatePresignedPutRequestFields]):
+    """
+    ```proto
+    message sphinx.CreatePresignedPutRequest
+    ```
+
+    Attributes:
+        project_id:
+            ```proto
+            string project_id = 1;
+            ```
+        content_type:
+            ```proto
+            optional string content_type = 2;
+            ```
+        filename:
+            ```proto
+            optional string filename = 3;
+            ```
+    """
+
+    __slots__ = ("project_id", "content_type", "filename")
+
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *,
+            project_id: str = "",
+            content_type: str | None = None,
+            filename: str | None = None,
+        ) -> None:
+            pass
+
+        project_id: str
+        content_type: str
+        filename: str
+
+_CreatePresignedPutResponseFields: TypeAlias = Literal["presigned_url", "ref", "required_headers"]
+
+class CreatePresignedPutResponse(Message[_CreatePresignedPutResponseFields]):
+    """
+    ```proto
+    message sphinx.CreatePresignedPutResponse
+    ```
+
+    Attributes:
+        presigned_url:
+            ```proto
+            string presigned_url = 1;
+            ```
+        ref:
+            ```proto
+            string ref = 2;
+            ```
+        required_headers:
+            Headers the client MUST include on the PUT request. The presigned URL
+            is signed against these exact header name/value pairs; omitting any of
+            them causes S3 to reject the upload with SignatureDoesNotMatch.
+
+            ```proto
+            map<string, string> required_headers = 3;
+            ```
+    """
+
+    __slots__ = ("presigned_url", "ref", "required_headers")
+
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *,
+            presigned_url: str = "",
+            ref: str = "",
+            required_headers: dict[str, str] | None = None,
+        ) -> None:
+            pass
+
+        presigned_url: str
+        ref: str
+        required_headers: dict[str, str]
+
+
+_DESC = file_desc(
+    b'\n\x14object_storage.proto\x12\x06sphinx"\xa1\x01\n\x19CreatePresignedPutRequest\x12\x1d\n\nproject_id\x18\x01 \x01(\tR\tprojectId\x12&\n\x0ccontent_type\x18\x02 \x01(\tH\x00R\x0bcontentType\x88\x01\x01\x12\x1f\n\x08filename\x18\x03 \x01(\tH\x01R\x08filename\x88\x01\x01B\x0f\n\r_content_typeB\x0b\n\t_filename"\xfb\x01\n\x1aCreatePresignedPutResponse\x12#\n\rpresigned_url\x18\x01 \x01(\tR\x0cpresignedUrl\x12\x10\n\x03ref\x18\x02 \x01(\tR\x03ref\x12b\n\x10required_headers\x18\x03 \x03(\x0b27.sphinx.CreatePresignedPutResponse.RequiredHeadersEntryR\x0frequiredHeaders\x1aB\n\x14RequiredHeadersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x028\x01b\x06proto3',
+    [],
+    {
+        "CreatePresignedPutRequest": CreatePresignedPutRequest,
+        "CreatePresignedPutResponse": CreatePresignedPutResponse,
+    },
+)
+
+
+def desc() -> DescFile:
+    """Returns the descriptor for the file `object_storage.proto`."""
+    return _DESC

sphinx_ai/gen/ra/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations