outerproduct 0.1.0__py3-none-any.whl

outerproduct/__init__.py

@@ -0,0 +1,153 @@
+"""OuterProduct — a Python SDK for training and explaining tabular ML models.
+
+The package exposes a small, flat surface organised around the following
+core concepts:
+
+- :class:`~outerproduct.dataset.Dataset` — a tabular dataset with an optional label column and
+  per-column schema (:class:`~outerproduct.dataset.Column`).
+- :class:`~outerproduct.connector.Connector` — a source of data (:class:`~outerproduct.connector.FileUploadConnector`,
+  :class:`~outerproduct.connector.S3Connector`, …) that produces a :class:`~outerproduct.dataset.Dataset`.
+- :class:`~outerproduct.trainer.Trainer` — orchestrates general-purpose training and
+  returns a :class:`~outerproduct.model.Model` via :meth:`~outerproduct.trainer.Trainer.configure` and
+  :meth:`~outerproduct.trainer.Trainer.run`.
+- :class:`~outerproduct.model.Model` — a trained predictor with :meth:`~outerproduct.model.Model.predict`.
+- :class:`~outerproduct.model.Predictor` — a duck-typed wrapper around an external
+  prediction endpoint, used to hand black-box models into
+  :func:`~outerproduct.reasoning.fit` for distillation.
+- :class:`~outerproduct.model.ReasoningModel` — a :class:`~outerproduct.model.Model` that also supports
+  :meth:`~outerproduct.model.ReasoningModel.explain`, :meth:`~outerproduct.model.ReasoningModel.interpret`,
+  :meth:`~outerproduct.model.ReasoningModel.scenario`, and :meth:`~outerproduct.model.ReasoningModel.segment`.
+  Produced by :func:`~outerproduct.reasoning.fit`.
+- :class:`~outerproduct.reasoning.Reasoning` / :class:`~outerproduct.reasoning.ReasoningTrace` — the typed results of
+  reasoning calls.
+
+Quickstart
+----------
+.. code-block:: python
+
+    import outerproduct as op
+
+    op.init(api_key="...")
+    connector = op.FileUploadConnector()
+    dataset = connector.upload("customers.csv", label_column="churn")
+    reasoning_model = op.reasoning.fit(dataset)
+    predictions = reasoning_model.predict(X_new)
+    reasoning = reasoning_model.explain(X_new)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from importlib.metadata import version as _pkg_version
+
+from outerproduct_http_types import ConnectorType
+
+from outerproduct import agentic
+from outerproduct.connector import (
+    Connector,
+    DatabricksConnector,
+    FileUploadConnector,
+    S3Connector,
+    SnowflakeConnector,
+)
+from outerproduct.dataset import Column, Dataset
+from outerproduct.job import Job, ReasoningJob, TrainerJob
+from outerproduct.model import Model, ReasoningModel, predict
+from outerproduct.reasoning import Reasoning, ReasoningTrace
+from outerproduct.trainer import (
+    Hardware,
+    Metric,
+    ModalHardware,
+    Trainer,
+)
+
+try:
+    __version__ = _pkg_version("outerproduct")
+except Exception:
+    __version__ = "unknown"
+
+
+@dataclass
+class _GlobalState:
+    """Module-level state populated by :func:`~outerproduct.init`. Internal."""
+
+    api_key: str | None = None
+    base_url: str | None = None
+    hardware: Hardware | None = None
+    extra: dict[str, object] = field(default_factory=dict)
+
+
+_state = _GlobalState()
+
+
+def init(
+    api_key: str | None = None,
+    *,
+    base_url: str | None = None,
+    hardware: Hardware | None = None,
+) -> None:
+    """Configure the OuterProduct SDK for the current Python process.
+
+    Call once near the top of your program. Subsequent operations
+    (training, prediction, uploads) read credentials, the API base URL,
+    and the default execution backend from the state recorded here.
+
+    Examples
+    --------
+    .. code-block:: python
+
+        import outerproduct as op
+        op.init(api_key="op_live_...", hardware=op.ModalHardware())
+
+    Parameters
+    ----------
+    api_key : str, optional
+        Bearer token used to authenticate against the OuterProduct API.
+        If omitted, the ``OUTERPRODUCT_API_KEY`` environment variable is
+        consulted at use time.
+    base_url : str, optional
+        Override for the API base URL. If omitted, the SDK uses the
+        production endpoint.
+    hardware : Hardware, optional
+        Default execution backend used by :class:`~outerproduct.trainer.Trainer`
+        and :func:`~outerproduct.reasoning.fit`. If omitted, the API
+        server falls back to a managed Modal worker.
+    """
+    _state.api_key = api_key
+    _state.base_url = base_url
+    _state.hardware = hardware
+
+
+__all__ = [
+    # Bootstrap
+    "init",
+    "__version__",
+    # Dataset
+    "Dataset",
+    "Column",
+    # Jobs
+    "Job",
+    "TrainerJob",
+    "ReasoningJob",
+    # Model
+    "Model",
+    "ReasoningModel",
+    "predict",
+    # Reasoning
+    "Reasoning",
+    "ReasoningTrace",
+    # Trainer
+    "Trainer",
+    "Hardware",
+    "ModalHardware",
+    "Metric",
+    # Connector
+    "Connector",
+    "ConnectorType",
+    "FileUploadConnector",
+    "S3Connector",
+    "SnowflakeConnector",
+    "DatabricksConnector",
+    # Agentic feature sourcing (submodule, accessed as op.agentic.*)
+    "agentic",
+]

outerproduct/agentic/__init__.py

@@ -0,0 +1,16 @@
+"""Agent-driven feature sourcing.
+
+Everything under :mod:`outerproduct.agentic` is part of an agent-driven flow
+that turns unstructured or semi-structured sources into typed tabular
+features. The boundary is the *flow*, not whether each individual function
+makes an LLM call — deterministic helpers (PDF parsing, batching, schema
+persistence) live here too because they support the same contract.
+
+Submodules
+----------
+- :mod:`outerproduct.agentic.documents` — files (PDF, image, text).
+"""
+
+from outerproduct.agentic import documents
+
+__all__ = ["documents"]

outerproduct/agentic/documents.py

@@ -0,0 +1,514 @@
+"""Document tabularization — agent-driven feature sourcing for files.
+
+One submodule of :mod:`outerproduct.agentic`. Treats files (PDF, image,
+text) as a feature source: induce a frozen schema of survey questions
+for the use case, then have the agent extract every document against
+that schema. The tabularized rows live in object storage on the
+OuterProduct backend; the SDK returns a :class:`DocumentDataset` that
+references that remote table by ``model_id`` and slots into
+:meth:`outerproduct.Trainer.configure` and
+:func:`outerproduct.reasoning.fit` exactly like any other uploaded
+:class:`~outerproduct.Dataset`.
+
+Typical flow::
+
+    docs = DocumentSet.from_directory("./invoices")
+    refs = upload(docs)                                  # one-time upload
+    schema = induce_schema(refs, use_case="audit", skill="invoice")
+    ds = tabularize(refs, schema)                        # remote DocumentDataset
+    trainer.run(ds)                                      # uses data_uploaded=True
+
+``upload`` is explicit so the same documents can feed both
+:func:`induce_schema` and :func:`tabularize` without re-uploading.
+For one-shot calls, ``induce_schema`` and ``tabularize`` also accept a
+:class:`DocumentSet` directly and upload internally.
+"""
+
+from __future__ import annotations
+
+import json
+import random as _random
+import re
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, Literal
+
+import httpx
+from outerproduct_http_types import (
+    AnswerType as _AnswerTypePayload,
+    CreateDocumentUploadRequest,
+    DocumentRef,
+    InduceSchemaRequest,
+    Question as QuestionPayload,
+    Schema as SchemaPayload,
+    TabularizeRequest,
+)
+
+from outerproduct.client import OuterProductClient
+from outerproduct.dataset import Column, Dataset
+
+DocumentMediaType = Literal[
+    "application/pdf",
+    "image/png",
+    "image/jpeg",
+    "image/gif",
+    "image/webp",
+    "text/plain",
+]
+
+
+class AnswerType(StrEnum):
+    """The legal answer shapes for a :class:`DocumentQuestion`."""
+
+    BOOLEAN = "boolean"
+    NUMBER = "number"
+    INTEGER = "integer"
+    ENUM = "enum"
+    MULTI_ENUM = "multi_enum"
+    DATE = "date"
+    DATE_RANGE = "date_range"
+    STRING = "string"
+    TEXT = "text"
+
+
+_ID_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+@dataclass(frozen=True)
+class DocumentQuestion:
+    """One survey question in a :class:`DocumentSchema`."""
+
+    id: str
+    question: str
+    answer_type: AnswerType
+    rationale: str | None = None
+    unit: str | None = None
+    enum: tuple[str, ...] | None = None
+
+    def __post_init__(self) -> None:
+        if not _ID_RE.match(self.id):
+            raise ValueError(f"DocumentQuestion.id={self.id!r} must be snake_case")
+        if self.answer_type in (AnswerType.ENUM, AnswerType.MULTI_ENUM):
+            if not self.enum:
+                raise ValueError(
+                    f"DocumentQuestion {self.id!r}: answer_type="
+                    f"{self.answer_type.value} requires non-empty `enum`"
+                )
+        elif self.enum is not None:
+            raise ValueError(
+                f"DocumentQuestion {self.id!r}: `enum` is only valid for "
+                "answer_type='enum' or 'multi_enum'"
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "id": self.id,
+            "question": self.question,
+            "answer_type": self.answer_type.value,
+        }
+        if self.rationale is not None:
+            out["rationale"] = self.rationale
+        if self.unit is not None:
+            out["unit"] = self.unit
+        if self.enum is not None:
+            out["enum"] = list(self.enum)
+        return out
+
+    @classmethod
+    def from_dict(cls, obj: dict[str, Any]) -> DocumentQuestion:
+        enum = obj.get("enum")
+        return cls(
+            id=obj["id"],
+            question=obj["question"],
+            answer_type=AnswerType(obj["answer_type"]),
+            rationale=obj.get("rationale"),
+            unit=obj.get("unit"),
+            enum=tuple(enum) if enum is not None else None,
+        )
+
+
+@dataclass
+class DocumentSchema:
+    """A frozen list of :class:`DocumentQuestion` for one skill and one
+    use case. Persist with :meth:`save`; reload with :meth:`load`."""
+
+    skill: str
+    use_case: str
+    questions: list[DocumentQuestion]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        seen: set[str] = set()
+        for q in self.questions:
+            if q.id in seen:
+                raise ValueError(f"DocumentSchema: duplicate question id {q.id!r}")
+            seen.add(q.id)
+
+    @property
+    def question_ids(self) -> list[str]:
+        return [q.id for q in self.questions]
+
+    def get(self, question_id: str) -> DocumentQuestion:
+        for q in self.questions:
+            if q.id == question_id:
+                return q
+        raise KeyError(question_id)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "skill": self.skill,
+            "use_case": self.use_case,
+            "questions": [q.to_dict() for q in self.questions],
+            "metadata": self.metadata,
+            "_format_version": 1,
+        }
+
+    @classmethod
+    def from_dict(cls, obj: dict[str, Any]) -> DocumentSchema:
+        return cls(
+            skill=obj["skill"],
+            use_case=obj["use_case"],
+            questions=[DocumentQuestion.from_dict(q) for q in obj["questions"]],
+            metadata=obj.get("metadata", {}),
+        )
+
+    def save(self, path: str | Path) -> None:
+        Path(path).write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: str | Path) -> DocumentSchema:
+        return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
+
+
+_PDF_SUFFIXES = {".pdf"}
+_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+_TEXT_SUFFIXES = {".txt", ".md"}
+_SUPPORTED_SUFFIXES = _PDF_SUFFIXES | _IMAGE_SUFFIXES | _TEXT_SUFFIXES
+_MIME_BY_SUFFIX: dict[str, DocumentMediaType] = {
+    ".pdf": "application/pdf",
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".txt": "text/plain",
+    ".md": "text/plain",
+}
+
+
+@dataclass(frozen=True)
+class Document:
+    """One document — raw bytes plus identity."""
+
+    document_id: str
+    bytes_: bytes
+    media_type: DocumentMediaType
+    source: Path | None = None
+
+    @classmethod
+    def from_path(cls, path: str | Path, *, document_id: str | None = None) -> Document:
+        p = Path(path)
+        suffix = p.suffix.lower()
+        if suffix not in _MIME_BY_SUFFIX:
+            raise ValueError(
+                f"Document.from_path: unsupported suffix {suffix!r} for {p}; "
+                f"supported: {sorted(_SUPPORTED_SUFFIXES)}"
+            )
+        return cls(
+            document_id=document_id or p.stem,
+            bytes_=p.read_bytes(),
+            media_type=_MIME_BY_SUFFIX[suffix],
+            source=p,
+        )
+
+    @classmethod
+    def from_text(cls, text: str, *, document_id: str) -> Document:
+        return cls(
+            document_id=document_id,
+            bytes_=text.encode("utf-8"),
+            media_type="text/plain",
+        )
+
+
+@dataclass
+class DocumentSet:
+    """An ordered collection of :class:`Document`."""
+
+    documents: list[Document] = field(default_factory=list)
+
+    @classmethod
+    def from_directory(
+        cls,
+        path: str | Path,
+        *,
+        glob: str = "*",
+        recursive: bool = False,
+    ) -> DocumentSet:
+        root = Path(path)
+        if not root.is_dir():
+            raise NotADirectoryError(
+                f"DocumentSet.from_directory: {root} is not a directory"
+            )
+        iterator = root.rglob(glob) if recursive else root.glob(glob)
+        docs = [
+            Document.from_path(p)
+            for p in sorted(iterator)
+            if p.is_file() and p.suffix.lower() in _SUPPORTED_SUFFIXES
+        ]
+        return cls(documents=docs)
+
+    @classmethod
+    def from_paths(cls, paths: Iterable[str | Path]) -> DocumentSet:
+        return cls(documents=[Document.from_path(p) for p in paths])
+
+    def sample(self, n: int, *, seed: int = 0) -> DocumentSet:
+        if n <= 0:
+            raise ValueError(f"sample: n must be positive, got {n}")
+        if n >= len(self.documents):
+            return DocumentSet(documents=list(self.documents))
+        rng = _random.Random(seed)
+        return DocumentSet(documents=rng.sample(self.documents, n))
+
+    def __iter__(self) -> Iterator[Document]:
+        return iter(self.documents)
+
+    def __len__(self) -> int:
+        return len(self.documents)
+
+    def __getitem__(self, index: int) -> Document:
+        return self.documents[index]
+
+    def __repr__(self) -> str:
+        return f"DocumentSet(n={len(self)})"
+
+
+class DocumentDataset(Dataset):
+    """A remote-handle :class:`Dataset` whose rows are tabularized
+    documents and columns are :class:`DocumentSchema` questions.
+
+    Holds no rows locally. The tabularized table lives on the
+    OuterProduct backend under ``model_id`` (the same path used by
+    pre-uploaded datasets), and the trainer/reasoning APIs fetch from
+    that location server-side via the standard ``data_uploaded=True``
+    wire mode.
+
+    Construct directly when you already have a tabularize ``model_id``,
+    or — more commonly — let :func:`tabularize` build one for you.
+    """
+
+    def __init__(
+        self,
+        *,
+        schema: DocumentSchema,
+        document_ids: list[str],
+        model_id: str,
+        label_column: str | None = None,
+        columns: list[Column] | None = None,
+    ) -> None:
+        import pandas as pd
+
+        empty = pd.DataFrame(columns=pd.Index([q.id for q in schema.questions]))
+        super().__init__(empty, label_column=label_column, columns=columns)
+        self._upload_id = model_id
+        self._schema = schema
+        self._document_ids = list(document_ids)
+
+    @property
+    def schema(self) -> DocumentSchema:
+        return self._schema
+
+    @property
+    def document_ids(self) -> list[str]:
+        return list(self._document_ids)
+
+    @property
+    def model_id(self) -> str:
+        """The tabularize-job ``model_id`` that backs this dataset on the server."""
+        assert self._upload_id is not None
+        return self._upload_id
+
+    @property
+    def n_samples(self) -> int:
+        return len(self._document_ids)
+
+
+# --------------------------------------------------------------------------- #
+# Wire conversions between the in-SDK data classes and the http-types models. #
+# --------------------------------------------------------------------------- #
+
+
+def _question_to_payload(q: DocumentQuestion) -> QuestionPayload:
+    return QuestionPayload(
+        id=q.id,
+        question=q.question,
+        answer_type=_AnswerTypePayload(q.answer_type.value),
+        rationale=q.rationale,
+        unit=q.unit,
+        enum=list(q.enum) if q.enum is not None else None,
+    )
+
+
+def _question_from_payload(p: QuestionPayload) -> DocumentQuestion:
+    return DocumentQuestion(
+        id=p.id,
+        question=p.question,
+        answer_type=AnswerType(p.answer_type.value),
+        rationale=p.rationale,
+        unit=p.unit,
+        enum=tuple(p.enum) if p.enum is not None else None,
+    )
+
+
+def _schema_to_payload(s: DocumentSchema) -> SchemaPayload:
+    return SchemaPayload(
+        skill=s.skill,
+        use_case=s.use_case,
+        questions=[_question_to_payload(q) for q in s.questions],
+        metadata=dict(s.metadata),
+    )
+
+
+def _schema_from_payload(p: SchemaPayload) -> DocumentSchema:
+    return DocumentSchema(
+        skill=p.skill,
+        use_case=p.use_case,
+        questions=[_question_from_payload(q) for q in p.questions],
+        metadata=dict(p.metadata),
+    )
+
+
+def _resolve_client(client: OuterProductClient | None) -> OuterProductClient:
+    return client if client is not None else OuterProductClient.from_credentials()
+
+
+def _coerce_to_refs(
+    documents: DocumentSet | list[DocumentRef],
+    client: OuterProductClient,
+) -> list[DocumentRef]:
+    """Accept either a :class:`DocumentSet` (upload internally) or
+    pre-uploaded :class:`DocumentRef` list (return as-is)."""
+    if isinstance(documents, DocumentSet):
+        return _upload_document_set(documents, client)
+    return list(documents)
+
+
+def _upload_document_set(
+    documents: DocumentSet, client: OuterProductClient
+) -> list[DocumentRef]:
+    refs: list[DocumentRef] = []
+    for doc in documents:
+        resp = client.uploads_api.create_document(
+            CreateDocumentUploadRequest(
+                document_id=doc.document_id, media_type=doc.media_type
+            )
+        )
+        put = httpx.put(
+            resp.upload_url,
+            content=doc.bytes_,
+            headers={"Content-Type": doc.media_type},
+        )
+        put.raise_for_status()
+        refs.append(
+            DocumentRef(
+                document_id=resp.document_id,
+                upload_key=resp.upload_key,
+                media_type=resp.media_type,
+            )
+        )
+    return refs
+
+
+# --------------------------------------------------------------------------- #
+# Public entry points                                                         #
+# --------------------------------------------------------------------------- #
+
+
+def upload(
+    documents: DocumentSet, *, client: OuterProductClient | None = None
+) -> list[DocumentRef]:
+    """Upload every document in ``documents`` via per-file presigned URLs.
+
+    Returns a list of :class:`DocumentRef` you can pass to
+    :func:`induce_schema` and :func:`tabularize` — uploading once and
+    reusing the refs avoids re-uploading the same bytes for both jobs.
+    """
+    op = _resolve_client(client)
+    return _upload_document_set(documents, op)
+
+
+def induce_schema(
+    documents: DocumentSet | list[DocumentRef],
+    *,
+    use_case: str,
+    skill: str,
+    client: OuterProductClient | None = None,
+) -> DocumentSchema:
+    """Run the agent over a sample of documents to produce a frozen schema.
+
+    Pass either a :class:`DocumentSet` (uploads internally) or a list of
+    :class:`DocumentRef` from a previous :func:`upload` call. Submits the
+    job, polls until completion, and returns the produced
+    :class:`DocumentSchema`.
+    """
+    op = _resolve_client(client)
+    refs = _coerce_to_refs(documents, op)
+    submission = op.agentic_documents_api.induce_schema(
+        InduceSchemaRequest(documents=refs, use_case=use_case, skill=skill)
+    )
+    op.wait_for(submission.model_id)
+    result = op.agentic_documents_api.get_schema(submission.model_id)
+    return _schema_from_payload(result.schema_)
+
+
+def tabularize(
+    documents: DocumentSet | list[DocumentRef],
+    schema: DocumentSchema,
+    *,
+    web_augmentation: bool = False,
+    concurrency: int = 1,
+    label_column: str | None = None,
+    client: OuterProductClient | None = None,
+) -> DocumentDataset:
+    """Extract every document against ``schema`` and return a :class:`DocumentDataset`.
+
+    The tabularized rows are written to object storage on the backend;
+    the returned :class:`DocumentDataset` is a remote handle that
+    :class:`~outerproduct.Trainer` and :func:`~outerproduct.reasoning.fit`
+    consume directly via the standard ``data_uploaded=True`` wire mode.
+    No rows are downloaded to the client.
+    """
+    op = _resolve_client(client)
+    refs = _coerce_to_refs(documents, op)
+    submission = op.agentic_documents_api.tabularize(
+        TabularizeRequest.model_validate(
+            {
+                "documents": [r.model_dump() for r in refs],
+                "schema": _schema_to_payload(schema).model_dump(),
+                "web_augmentation": web_augmentation,
+                "concurrency": concurrency,
+            }
+        )
+    )
+    op.wait_for(submission.model_id)
+    result = op.agentic_documents_api.get_table(submission.model_id)
+    return DocumentDataset(
+        schema=_schema_from_payload(result.schema_),
+        document_ids=list(result.document_ids),
+        model_id=result.model_id,
+        label_column=label_column,
+    )
+
+
+__all__ = [
+    "AnswerType",
+    "Document",
+    "DocumentDataset",
+    "DocumentQuestion",
+    "DocumentSchema",
+    "DocumentSet",
+    "induce_schema",
+    "tabularize",
+    "upload",
+]