docforge-sdk 0.2.0__tar.gz → 0.4.0__tar.gz

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docforge-sdk
-Version: 0.2.0
+Version: 0.4.0
 Summary: Typed async + sync Python client for the DocForge REST API (standalone, zero server deps).
 Project-URL: Homepage, https://github.com/Florian-BARRE/docforge
 Project-URL: Repository, https://github.com/Florian-BARRE/docforge
@@ -189,6 +189,7 @@ are identical.
 | `create(CreateCollectionRequest)` | `CollectionModel` | Create a collection (contract). |
 | `update(collection_id, UpdateCollectionRequest)` | `CollectionModel` | Patch name / formats / fields / pipelines. |
 | `delete(collection_id)` | `None` | Delete a collection. |
+| `storage(collection_id)` | `CollectionStorageResponse` | Material storage footprint per store (S3 exact, Postgres/Qdrant estimated) + per-document breakdown. |
 
 ### `documents`
 | Method | Returns | Purpose |
@@ -271,6 +272,12 @@ with Client("http://localhost:10040", api_token="df_...") as client:
     print(collection.id)
 ```
 
+Key `CreateCollectionRequest` fields beyond the schema: `max_file_size_bytes` (bytes) and
+`job_timeout_seconds` (`float | None`, seconds) — the whole-ingest-job wall-clock budget for that
+collection; `None` (the default) inherits the worker's global job-timeout default. Same field, same
+semantics on `CollectionModel` (read) and `UpdateCollectionRequest` (write; there, omitting it
+leaves the current value unchanged, a set value overrides it).
+
 ### Upload a document and wait for ingestion
 
 `upload` returns immediately with a `job_id`; ingestion runs asynchronously on the worker. Poll the
@@ -338,6 +345,20 @@ with Client("http://localhost:10040", api_token="df_...") as client:
     client.explorer.set_chunk_enabled(chunks[0].chunk_id, enabled=False)
 ```
 
+### Measure a collection's storage footprint
+
+`storage` reports the material footprint per store — S3 bytes are exact (deduped), Postgres/Qdrant
+bytes are estimates (each section flags this via its own `estimated`) — plus a per-document
+breakdown sorted heaviest first.
+
+```python
+with Client("http://localhost:10040", api_token="df_...") as client:
+    footprint = client.collections.storage(collection_id)
+    print(footprint.grand_total_bytes, footprint.s3.physical_unique_bytes)
+    for doc in footprint.documents[:5]:
+        print(doc.filename, doc.total_bytes)
+```
+
 ### Manage API keys
 
 Mint a scoped, expiring key (the plaintext is shown **once**, on creation):
@@ -424,6 +445,7 @@ from docforge_sdk import (
     UpdateCollectionRequest,
     FieldSpec,
     FieldType,
+    CollectionStorageResponse,
     # documents / explorer
     UploadAccepted,
     DocumentDetail,
@@ -465,7 +487,7 @@ server's OpenAPI on every change, so a published version is coherent with the AP
 version in production:
 
 ```bash
-pip install "docforge-sdk==0.1.0"
+pip install "docforge-sdk==0.3.0"
 ```
 
 ## License

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/README.md

@@ -166,6 +166,7 @@ are identical.
 | `create(CreateCollectionRequest)` | `CollectionModel` | Create a collection (contract). |
 | `update(collection_id, UpdateCollectionRequest)` | `CollectionModel` | Patch name / formats / fields / pipelines. |
 | `delete(collection_id)` | `None` | Delete a collection. |
+| `storage(collection_id)` | `CollectionStorageResponse` | Material storage footprint per store (S3 exact, Postgres/Qdrant estimated) + per-document breakdown. |
 
 ### `documents`
 | Method | Returns | Purpose |
@@ -248,6 +249,12 @@ with Client("http://localhost:10040", api_token="df_...") as client:
     print(collection.id)
 ```
 
+Key `CreateCollectionRequest` fields beyond the schema: `max_file_size_bytes` (bytes) and
+`job_timeout_seconds` (`float | None`, seconds) — the whole-ingest-job wall-clock budget for that
+collection; `None` (the default) inherits the worker's global job-timeout default. Same field, same
+semantics on `CollectionModel` (read) and `UpdateCollectionRequest` (write; there, omitting it
+leaves the current value unchanged, a set value overrides it).
+
 ### Upload a document and wait for ingestion
 
 `upload` returns immediately with a `job_id`; ingestion runs asynchronously on the worker. Poll the
@@ -315,6 +322,20 @@ with Client("http://localhost:10040", api_token="df_...") as client:
     client.explorer.set_chunk_enabled(chunks[0].chunk_id, enabled=False)
 ```
 
+### Measure a collection's storage footprint
+
+`storage` reports the material footprint per store — S3 bytes are exact (deduped), Postgres/Qdrant
+bytes are estimates (each section flags this via its own `estimated`) — plus a per-document
+breakdown sorted heaviest first.
+
+```python
+with Client("http://localhost:10040", api_token="df_...") as client:
+    footprint = client.collections.storage(collection_id)
+    print(footprint.grand_total_bytes, footprint.s3.physical_unique_bytes)
+    for doc in footprint.documents[:5]:
+        print(doc.filename, doc.total_bytes)
+```
+
 ### Manage API keys
 
 Mint a scoped, expiring key (the plaintext is shown **once**, on creation):
@@ -401,6 +422,7 @@ from docforge_sdk import (
     UpdateCollectionRequest,
     FieldSpec,
     FieldType,
+    CollectionStorageResponse,
     # documents / explorer
     UploadAccepted,
     DocumentDetail,
@@ -442,7 +464,7 @@ server's OpenAPI on every change, so a published version is coherent with the AP
 version in production:
 
 ```bash
-pip install "docforge-sdk==0.1.0"
+pip install "docforge-sdk==0.3.0"
 ```
 
 ## License

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/docforge_sdk/__init__.py

@@ -80,6 +80,15 @@ from .models.pipelines import (
 # ------------------- Search models ------------------- #
 from .models.search import BlockLocation, SearchHit, SearchRequest, SearchResponse, SearchTarget
 
+# ------------------- Storage models ------------------- #
+from .models.storage import (
+    CollectionStorageResponse,
+    DocumentStorageModel,
+    PostgresFootprintModel,
+    QdrantFootprintModel,
+    S3FootprintModel,
+)
+
 # ------------------- Public API ------------------- #
 __all__ = [
     "__version__",
@@ -150,6 +159,12 @@ __all__ = [
     "EditResponse",
     "StageViewResponse",
     "StageApplyResponse",
+    # Storage
+    "S3FootprintModel",
+    "PostgresFootprintModel",
+    "QdrantFootprintModel",
+    "DocumentStorageModel",
+    "CollectionStorageResponse",
     # Exceptions
     "DocForgeError",
     "APIConnectionError",

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/docforge_sdk/_version.py

@@ -2,4 +2,4 @@
 # Single source of truth for the package version. Hatchling reads ``__version__`` from this file to
 # populate the distribution metadata (see ``[tool.hatch.version]`` in pyproject.toml).
 
-__version__ = "0.2.0"
+__version__ = "0.4.0"

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/docforge_sdk/models/__init__.py

@@ -64,6 +64,15 @@ from .pipelines import (
 # ------------------- Search models ------------------- #
 from .search import BlockLocation, SearchHit, SearchRequest, SearchResponse, SearchTarget
 
+# ------------------- Storage models ------------------- #
+from .storage import (
+    CollectionStorageResponse,
+    DocumentStorageModel,
+    PostgresFootprintModel,
+    QdrantFootprintModel,
+    S3FootprintModel,
+)
+
 # ------------------- Public API ------------------- #
 __all__ = [
     # Shared vocabulary
@@ -130,4 +139,10 @@ __all__ = [
     "EditResponse",
     "StageViewResponse",
     "StageApplyResponse",
+    # Storage
+    "S3FootprintModel",
+    "PostgresFootprintModel",
+    "QdrantFootprintModel",
+    "DocumentStorageModel",
+    "CollectionStorageResponse",
 ]

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/docforge_sdk/models/collections.py

@@ -56,6 +56,8 @@ class CollectionModel(BaseModel):
         name (str): Unique human name.
         supported_formats (list[str]): Accepted upload extensions (e.g. pdf).
         max_file_size_bytes (int): Upload size ceiling, bytes.
+        job_timeout_seconds (float | None): Per-collection whole-ingest-job wall-clock budget,
+            seconds. None = inherit the worker's global WORKER_JOB_TIMEOUT_SECONDS default.
         needs_reindex (bool): True when a config change requires reindexing.
         created_at (datetime | None): Creation timestamp.
         pipeline (dict[str, Any]): The ingestion pipeline blob (the graph).
@@ -67,6 +69,14 @@ class CollectionModel(BaseModel):
     name: str = Field(description="Unique human name.")
     supported_formats: list[str] = Field(description="Accepted upload extensions (e.g. pdf).")
     max_file_size_bytes: int = Field(description="Upload size ceiling, bytes.")
+    job_timeout_seconds: float | None = Field(
+        default=None,
+        gt=0,
+        description=(
+            "Per-collection whole-ingest-job wall-clock budget, seconds. None = inherit the "
+            "worker's global WORKER_JOB_TIMEOUT_SECONDS default."
+        ),
+    )
     needs_reindex: bool = Field(description="True when a config change requires reindexing.")
     created_at: datetime | None = Field(default=None, description="Creation timestamp.")
     pipeline: dict[str, Any] = Field(description="The ingestion pipeline blob (the graph).")
@@ -84,6 +94,8 @@ class CreateCollectionRequest(BaseModel):
         name (str): Unique human name.
         supported_formats (list[str]): Accepted upload extensions (e.g. pdf).
         max_file_size_bytes (int): Upload size ceiling, bytes.
+        job_timeout_seconds (float | None): Per-collection whole-ingest-job wall-clock budget,
+            seconds. None = inherit the worker's global WORKER_JOB_TIMEOUT_SECONDS default.
         fields (list[FieldSpec]): The FULL schema, declared up front (vector space is fixed).
         pipeline (dict[str, Any] | None): The pipeline blob; omitted → the product default.
     """
@@ -91,6 +103,14 @@ class CreateCollectionRequest(BaseModel):
     name: str = Field(description="Unique human name.")
     supported_formats: list[str] = Field(description="Accepted upload extensions (e.g. pdf).")
     max_file_size_bytes: int = Field(description="Upload size ceiling, bytes.")
+    job_timeout_seconds: float | None = Field(
+        default=None,
+        gt=0,
+        description=(
+            "Per-collection whole-ingest-job wall-clock budget, seconds. None = inherit the "
+            "worker's global WORKER_JOB_TIMEOUT_SECONDS default."
+        ),
+    )
     fields: list[FieldSpec] = Field(
         default_factory=list,
         description="The FULL schema, declared up front (vector space is fixed at creation).",
@@ -114,6 +134,8 @@ class UpdateCollectionRequest(BaseModel):
         name (str | None): New unique name.
         supported_formats (list[str] | None): New accepted upload extensions.
         max_file_size_bytes (int | None): New size ceiling, bytes.
+        job_timeout_seconds (float | None): New per-collection whole-ingest-job wall-clock
+            budget, seconds. Omitted = leave the current value unchanged.
         fields (list[FieldSpec] | None): The TARGET schema (diffed by field name).
         pipeline (dict[str, Any] | None): New pipeline blob (validated before storage).
         search (dict[str, Any] | None): New search graph blob ({} = stock default).
@@ -125,6 +147,14 @@ class UpdateCollectionRequest(BaseModel):
         default=None, description="New accepted upload extensions."
     )
     max_file_size_bytes: int | None = Field(default=None, description="New size ceiling, bytes.")
+    job_timeout_seconds: float | None = Field(
+        default=None,
+        gt=0,
+        description=(
+            "New per-collection whole-ingest-job wall-clock budget, seconds. Omitted = leave the "
+            "current value unchanged; a set value overrides the global WORKER_JOB_TIMEOUT_SECONDS."
+        ),
+    )
     fields: list[FieldSpec] | None = Field(
         default=None,
         description="The TARGET schema (diffed by field name; omitted fields are removed).",

docforge_sdk-0.4.0/docforge_sdk/models/storage.py

@@ -0,0 +1,149 @@
+# ====== Code Summary ======
+# Response models for the collection storage-footprint endpoint, mirrored field-for-field from the
+# DocForge backend router models (S3FootprintModel / PostgresFootprintModel / QdrantFootprintModel /
+# DocumentStorageModel / CollectionStorageResponse). S3 bytes are EXACT; Postgres and Qdrant bytes are
+# ESTIMATES — each section carries its own ``estimated`` flag.
+
+# ====== Third-Party Library Imports ======
+from pydantic import BaseModel, Field
+
+
+class S3FootprintModel(BaseModel):
+    """
+    EXACT S3 bytes from the content-addressed blob registry (``estimated`` is always false).
+
+    Attributes:
+        original_bytes (int): Uploaded source file bytes.
+        rendered_bytes (int): Derived-blob bytes (canonical PDF, page renders, crops).
+        total_bytes (int): Logical bytes (original + rendered).
+        physical_unique_bytes (int): Deduped disk cost — a blob shared across documents counts
+            once (<= total).
+        estimated (bool): Always false: S3 bytes are measured exactly.
+    """
+
+    original_bytes: int = Field(description="Uploaded source file bytes.")
+    rendered_bytes: int = Field(
+        description="Derived-blob bytes (canonical PDF, page renders, crops)."
+    )
+    total_bytes: int = Field(description="Logical bytes (original + rendered).")
+    physical_unique_bytes: int = Field(
+        description="Deduped disk cost — a blob shared across documents counts once (<= total)."
+    )
+    estimated: bool = Field(description="Always false: S3 bytes are measured exactly.")
+
+
+class PostgresFootprintModel(BaseModel):
+    """
+    ESTIMATED Postgres row bytes via ``pg_column_size`` (excludes index/TOAST/bloat).
+
+    Attributes:
+        documents_bytes (int): ``document`` + ``page`` rows.
+        ir_blocks_bytes (int): ``block`` + ``block_table`` + ``block_figure`` rows.
+        enrichment_bytes (int): ``block_enrichment`` + ``enrichment_attempt`` rows.
+        chunks_bytes (int): ``chunk`` + ``chunk_block`` + ``chunk_metadata`` + ``entity_mention`` rows.
+        metadata_bytes (int): ``document_metadata`` rows.
+        observability_bytes (int): ``job`` + ``job_stage_event`` rows.
+        total_bytes (int): Sum of every bucket.
+        estimated (bool): Always true: real row bytes, no index/TOAST/bloat.
+    """
+
+    documents_bytes: int = Field(description="``document`` + ``page`` rows.")
+    ir_blocks_bytes: int = Field(description="``block`` + ``block_table`` + ``block_figure`` rows.")
+    enrichment_bytes: int = Field(description="``block_enrichment`` + ``enrichment_attempt`` rows.")
+    chunks_bytes: int = Field(
+        description="``chunk`` + ``chunk_block`` + ``chunk_metadata`` + ``entity_mention`` rows."
+    )
+    metadata_bytes: int = Field(description="``document_metadata`` rows.")
+    observability_bytes: int = Field(description="``job`` + ``job_stage_event`` rows.")
+    total_bytes: int = Field(description="Sum of every bucket.")
+    estimated: bool = Field(description="Always true: real row bytes, no index/TOAST/bloat.")
+
+
+class QdrantFootprintModel(BaseModel):
+    """
+    ESTIMATED vector-store bytes (points x declared shape — excludes HNSW index overhead).
+
+    Attributes:
+        points (int): Point count (collection total, or a document's points).
+        dense_bytes (int): On-disk float32 dense bytes, summed per named vector weighted by its
+            carrier count. Excludes the int8 quantized RAM-resident copy — this is disk, not RAM.
+        sparse_bytes (int): ``points * avg_sparse_entries * 8`` (int32 index + float32 value).
+        payload_bytes (int): ``points * avg_payload_json_bytes``.
+        total_bytes (int): Sum of dense + sparse + payload.
+        estimated (bool): Always true: count-based, excludes index overhead.
+    """
+
+    points: int = Field(description="Point count (collection total, or a document's points).")
+    dense_bytes: int = Field(
+        description=(
+            "On-disk float32 dense bytes, summed per named vector weighted by its carrier count "
+            "(content_dense on every point, each meta vector only on its field's documents). "
+            "Excludes the int8 quantized RAM-resident copy — this is disk, not RAM."
+        )
+    )
+    sparse_bytes: int = Field(
+        description="``points x avg_sparse_entries x 8`` (int32 index + float32 value)."
+    )
+    payload_bytes: int = Field(description="``points x avg_payload_json_bytes``.")
+    total_bytes: int = Field(description="Sum of dense + sparse + payload.")
+    estimated: bool = Field(description="Always true: count-based, excludes index overhead.")
+
+
+class DocumentStorageModel(BaseModel):
+    """
+    One document's footprint across the three stores.
+
+    Attributes:
+        document_id (str): The document's UUID.
+        filename (str): The document's display name.
+        s3 (S3FootprintModel): EXACT S3 bytes.
+        postgres (PostgresFootprintModel): ESTIMATED Postgres row bytes.
+        qdrant (QdrantFootprintModel): ESTIMATED vector-store bytes.
+        total_bytes (int): S3 (logical) + Postgres + Qdrant.
+    """
+
+    document_id: str = Field(description="The document's UUID.")
+    filename: str = Field(description="The document's display name.")
+    s3: S3FootprintModel = Field(description="EXACT S3 bytes.")
+    postgres: PostgresFootprintModel = Field(description="ESTIMATED Postgres row bytes.")
+    qdrant: QdrantFootprintModel = Field(description="ESTIMATED vector-store bytes.")
+    total_bytes: int = Field(description="S3 (logical) + Postgres + Qdrant.")
+
+
+class CollectionStorageResponse(BaseModel):
+    """
+    A collection's material footprint per store, plus the per-document breakdown (heaviest first).
+
+    S3 bytes are EXACT; Postgres and Qdrant bytes are ESTIMATES (each section flags this via its own
+    ``estimated``). ``grand_total_bytes`` uses the DEDUPED S3 disk cost (``physical_unique_bytes``),
+    so it reflects real hardware rather than the logical per-document sum.
+
+    Attributes:
+        collection_id (str): The measured collection's UUID.
+        s3 (S3FootprintModel): EXACT S3 totals (logical + deduped physical).
+        postgres (PostgresFootprintModel): ESTIMATED Postgres row bytes.
+        qdrant (QdrantFootprintModel): ESTIMATED vector-store bytes.
+        grand_total_bytes (int): Material footprint — S3 physical_unique + Postgres + Qdrant.
+        documents (list[DocumentStorageModel]): Per-document breakdown, sorted by total bytes
+            descending (doubles as top-N).
+    """
+
+    collection_id: str = Field(description="The measured collection's UUID.")
+    s3: S3FootprintModel = Field(description="EXACT S3 totals (logical + deduped physical).")
+    postgres: PostgresFootprintModel = Field(description="ESTIMATED Postgres row bytes.")
+    qdrant: QdrantFootprintModel = Field(description="ESTIMATED vector-store bytes.")
+    grand_total_bytes: int = Field(
+        description="Material footprint — S3 physical_unique + Postgres + Qdrant."
+    )
+    documents: list[DocumentStorageModel] = Field(
+        description="Per-document breakdown, sorted by total bytes descending (doubles as top-N)."
+    )
+
+
+__all__ = [
+    "S3FootprintModel",
+    "PostgresFootprintModel",
+    "QdrantFootprintModel",
+    "DocumentStorageModel",
+    "CollectionStorageResponse",
+]

{docforge_sdk-0.2.0 → docforge_sdk-0.4.0}/docforge_sdk/resources/collections.py

@@ -10,6 +10,7 @@ from ..models.collections import (
     CreateCollectionRequest,
     UpdateCollectionRequest,
 )
+from ..models.storage import CollectionStorageResponse
 from ._base import AsyncResource, SyncResource, _ResourceMixin
 
 
@@ -80,6 +81,18 @@ class _CollectionsSpecs(_ResourceMixin):
         """
         return RequestSpec("DELETE", f"{self._COLLECTIONS_PATH}/{collection_id}")
 
+    def _storage_spec(self, collection_id: str) -> RequestSpec:
+        """
+        Build the spec for measuring a collection's material storage footprint.
+
+        Args:
+            collection_id (str): The collection's UUID.
+
+        Returns:
+            RequestSpec: A GET on the collection's storage sub-resource.
+        """
+        return RequestSpec("GET", f"{self._COLLECTIONS_PATH}/{collection_id}/storage")
+
 
 class AsyncCollections(AsyncResource, _CollectionsSpecs):
     """Asynchronous collection management (list / get / create / update / delete)."""
@@ -141,6 +154,20 @@ class AsyncCollections(AsyncResource, _CollectionsSpecs):
         """
         return await self._transport.request(self._delete_spec(collection_id), type(None))
 
+    async def storage(self, collection_id: str) -> CollectionStorageResponse:
+        """
+        Measure a collection's material footprint per store (S3 exact, Postgres/Qdrant estimated).
+
+        Args:
+            collection_id (str): The collection's UUID.
+
+        Returns:
+            CollectionStorageResponse: Per-store totals + the per-document breakdown, heaviest first.
+        """
+        return await self._transport.request(
+            self._storage_spec(collection_id), CollectionStorageResponse
+        )
+
 
 class SyncCollections(SyncResource, _CollectionsSpecs):
     """Synchronous collection management (list / get / create / update / delete)."""
@@ -200,5 +227,17 @@ class SyncCollections(SyncResource, _CollectionsSpecs):
         """
         return self._transport.request(self._delete_spec(collection_id), type(None))
 
+    def storage(self, collection_id: str) -> CollectionStorageResponse:
+        """
+        Measure a collection's material footprint per store (S3 exact, Postgres/Qdrant estimated).
+
+        Args:
+            collection_id (str): The collection's UUID.
+
+        Returns:
+            CollectionStorageResponse: Per-store totals + the per-document breakdown, heaviest first.
+        """
+        return self._transport.request(self._storage_spec(collection_id), CollectionStorageResponse)
+
 
 __all__ = ["AsyncCollections", "SyncCollections"]