docforge-sdk 0.3.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/PKG-INFO +26 -3
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/README.md +24 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/__init__.py +17 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_version.py +1 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/__init__.py +17 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/jobs.py +63 -3
- docforge_sdk-0.7.0/docforge_sdk/models/storage.py +149 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/collections.py +39 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/jobs.py +47 -1
- docforge_sdk-0.7.0/tests/openapi_snapshot.json +6888 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/parity_map.py +22 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_collections.py +69 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_jobs.py +45 -1
- docforge_sdk-0.3.0/tests/openapi_snapshot.json +0 -1
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/.gitignore +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/.python-version +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/LICENSE +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_exceptions.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_requestspec.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_transport_async.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_transport_base.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/_transport_sync.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/client.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/_shared.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/auth.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/blobs.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/collections.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/documents.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/explorer.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/health.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/ir.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/pipelines.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/models/search.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/py.typed +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/__init__.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/_base.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/auth.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/blobs.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/documents.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/explorer.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/health.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/pipelines.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/docforge_sdk/resources/search.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/pyproject.toml +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/__init__.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/check_schema_drift.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/live/__init__.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/live/test_openapi_parity_live.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/__init__.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_auth.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_blobs.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_documents.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_explorer.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_health.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_models_offline_parity.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_pipelines.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_resource_parity.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_search.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/tests/unit/test_transport.py +0 -0
- {docforge_sdk-0.3.0 → docforge_sdk-0.7.0}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: docforge-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Typed async + sync Python client for the DocForge REST API (standalone, zero server deps).
|
|
5
5
|
Project-URL: Homepage, https://github.com/Florian-BARRE/docforge
|
|
6
6
|
Project-URL: Repository, https://github.com/Florian-BARRE/docforge
|
|
@@ -189,6 +189,7 @@ are identical.
|
|
|
189
189
|
| `create(CreateCollectionRequest)` | `CollectionModel` | Create a collection (contract). |
|
|
190
190
|
| `update(collection_id, UpdateCollectionRequest)` | `CollectionModel` | Patch name / formats / fields / pipelines. |
|
|
191
191
|
| `delete(collection_id)` | `None` | Delete a collection. |
|
|
192
|
+
| `storage(collection_id)` | `CollectionStorageResponse` | Material storage footprint per store (S3 exact, Postgres/Qdrant estimated) + per-document breakdown. |
|
|
192
193
|
|
|
193
194
|
### `documents`
|
|
194
195
|
| Method | Returns | Purpose |
|
|
@@ -220,6 +221,7 @@ are identical.
|
|
|
220
221
|
| `get(job_id)` | `JobStatus` | One job's status + progress. |
|
|
221
222
|
| `get_events(job_id)` | `JobTrace` | Per-stage event trace. |
|
|
222
223
|
| `live_workers()` | `WorkersLive` | Currently active workers. |
|
|
224
|
+
| `cancel(job_id, force=False)` | `CancelResult` | Stop a job — cooperative by default, immediate with `force=True`. |
|
|
223
225
|
|
|
224
226
|
### `blobs`
|
|
225
227
|
| Method | Returns | Purpose |
|
|
@@ -271,6 +273,12 @@ with Client("http://localhost:10040", api_token="df_...") as client:
|
|
|
271
273
|
print(collection.id)
|
|
272
274
|
```
|
|
273
275
|
|
|
276
|
+
Key `CreateCollectionRequest` fields beyond the schema: `max_file_size_bytes` (bytes) and
|
|
277
|
+
`job_timeout_seconds` (`float | None`, seconds) — the whole-ingest-job wall-clock budget for that
|
|
278
|
+
collection; `None` (the default) inherits the worker's global job-timeout default. Same field, same
|
|
279
|
+
semantics on `CollectionModel` (read) and `UpdateCollectionRequest` (write; there, omitting it
|
|
280
|
+
leaves the current value unchanged, a set value overrides it).
|
|
281
|
+
|
|
274
282
|
### Upload a document and wait for ingestion
|
|
275
283
|
|
|
276
284
|
`upload` returns immediately with a `job_id`; ingestion runs asynchronously on the worker. Poll the
|
|
@@ -338,6 +346,20 @@ with Client("http://localhost:10040", api_token="df_...") as client:
|
|
|
338
346
|
client.explorer.set_chunk_enabled(chunks[0].chunk_id, enabled=False)
|
|
339
347
|
```
|
|
340
348
|
|
|
349
|
+
### Measure a collection's storage footprint
|
|
350
|
+
|
|
351
|
+
`storage` reports the material footprint per store — S3 bytes are exact (deduped), Postgres/Qdrant
|
|
352
|
+
bytes are estimates (each section flags this via its own `estimated`) — plus a per-document
|
|
353
|
+
breakdown sorted heaviest first.
|
|
354
|
+
|
|
355
|
+
```python
|
|
356
|
+
with Client("http://localhost:10040", api_token="df_...") as client:
|
|
357
|
+
footprint = client.collections.storage(collection_id)
|
|
358
|
+
print(footprint.grand_total_bytes, footprint.s3.physical_unique_bytes)
|
|
359
|
+
for doc in footprint.documents[:5]:
|
|
360
|
+
print(doc.filename, doc.total_bytes)
|
|
361
|
+
```
|
|
362
|
+
|
|
341
363
|
### Manage API keys
|
|
342
364
|
|
|
343
365
|
Mint a scoped, expiring key (the plaintext is shown **once**, on creation):
|
|
@@ -424,6 +446,7 @@ from docforge_sdk import (
|
|
|
424
446
|
UpdateCollectionRequest,
|
|
425
447
|
FieldSpec,
|
|
426
448
|
FieldType,
|
|
449
|
+
CollectionStorageResponse,
|
|
427
450
|
# documents / explorer
|
|
428
451
|
UploadAccepted,
|
|
429
452
|
DocumentDetail,
|
|
@@ -465,7 +488,7 @@ server's OpenAPI on every change, so a published version is coherent with the AP
|
|
|
465
488
|
version in production:
|
|
466
489
|
|
|
467
490
|
```bash
|
|
468
|
-
pip install "docforge-sdk==0.
|
|
491
|
+
pip install "docforge-sdk==0.3.0"
|
|
469
492
|
```
|
|
470
493
|
|
|
471
494
|
## License
|
|
@@ -166,6 +166,7 @@ are identical.
|
|
|
166
166
|
| `create(CreateCollectionRequest)` | `CollectionModel` | Create a collection (contract). |
|
|
167
167
|
| `update(collection_id, UpdateCollectionRequest)` | `CollectionModel` | Patch name / formats / fields / pipelines. |
|
|
168
168
|
| `delete(collection_id)` | `None` | Delete a collection. |
|
|
169
|
+
| `storage(collection_id)` | `CollectionStorageResponse` | Material storage footprint per store (S3 exact, Postgres/Qdrant estimated) + per-document breakdown. |
|
|
169
170
|
|
|
170
171
|
### `documents`
|
|
171
172
|
| Method | Returns | Purpose |
|
|
@@ -197,6 +198,7 @@ are identical.
|
|
|
197
198
|
| `get(job_id)` | `JobStatus` | One job's status + progress. |
|
|
198
199
|
| `get_events(job_id)` | `JobTrace` | Per-stage event trace. |
|
|
199
200
|
| `live_workers()` | `WorkersLive` | Currently active workers. |
|
|
201
|
+
| `cancel(job_id, force=False)` | `CancelResult` | Stop a job — cooperative by default, immediate with `force=True`. |
|
|
200
202
|
|
|
201
203
|
### `blobs`
|
|
202
204
|
| Method | Returns | Purpose |
|
|
@@ -248,6 +250,12 @@ with Client("http://localhost:10040", api_token="df_...") as client:
|
|
|
248
250
|
print(collection.id)
|
|
249
251
|
```
|
|
250
252
|
|
|
253
|
+
Key `CreateCollectionRequest` fields beyond the schema: `max_file_size_bytes` (bytes) and
|
|
254
|
+
`job_timeout_seconds` (`float | None`, seconds) — the whole-ingest-job wall-clock budget for that
|
|
255
|
+
collection; `None` (the default) inherits the worker's global job-timeout default. Same field, same
|
|
256
|
+
semantics on `CollectionModel` (read) and `UpdateCollectionRequest` (write; there, omitting it
|
|
257
|
+
leaves the current value unchanged, a set value overrides it).
|
|
258
|
+
|
|
251
259
|
### Upload a document and wait for ingestion
|
|
252
260
|
|
|
253
261
|
`upload` returns immediately with a `job_id`; ingestion runs asynchronously on the worker. Poll the
|
|
@@ -315,6 +323,20 @@ with Client("http://localhost:10040", api_token="df_...") as client:
|
|
|
315
323
|
client.explorer.set_chunk_enabled(chunks[0].chunk_id, enabled=False)
|
|
316
324
|
```
|
|
317
325
|
|
|
326
|
+
### Measure a collection's storage footprint
|
|
327
|
+
|
|
328
|
+
`storage` reports the material footprint per store — S3 bytes are exact (deduped), Postgres/Qdrant
|
|
329
|
+
bytes are estimates (each section flags this via its own `estimated`) — plus a per-document
|
|
330
|
+
breakdown sorted heaviest first.
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
with Client("http://localhost:10040", api_token="df_...") as client:
|
|
334
|
+
footprint = client.collections.storage(collection_id)
|
|
335
|
+
print(footprint.grand_total_bytes, footprint.s3.physical_unique_bytes)
|
|
336
|
+
for doc in footprint.documents[:5]:
|
|
337
|
+
print(doc.filename, doc.total_bytes)
|
|
338
|
+
```
|
|
339
|
+
|
|
318
340
|
### Manage API keys
|
|
319
341
|
|
|
320
342
|
Mint a scoped, expiring key (the plaintext is shown **once**, on creation):
|
|
@@ -401,6 +423,7 @@ from docforge_sdk import (
|
|
|
401
423
|
UpdateCollectionRequest,
|
|
402
424
|
FieldSpec,
|
|
403
425
|
FieldType,
|
|
426
|
+
CollectionStorageResponse,
|
|
404
427
|
# documents / explorer
|
|
405
428
|
UploadAccepted,
|
|
406
429
|
DocumentDetail,
|
|
@@ -442,7 +465,7 @@ server's OpenAPI on every change, so a published version is coherent with the AP
|
|
|
442
465
|
version in production:
|
|
443
466
|
|
|
444
467
|
```bash
|
|
445
|
-
pip install "docforge-sdk==0.
|
|
468
|
+
pip install "docforge-sdk==0.3.0"
|
|
446
469
|
```
|
|
447
470
|
|
|
448
471
|
## License
|
|
@@ -64,7 +64,7 @@ from .models.health import HealthStatus
|
|
|
64
64
|
from .models.ir import DocumentIRModel, IRBlock, IREnrichment, IRFigure, IRTable
|
|
65
65
|
|
|
66
66
|
# ------------------- Jobs models ------------------- #
|
|
67
|
-
from .models.jobs import JobEvent, JobStatus, JobTrace, WorkerActivity, WorkersLive
|
|
67
|
+
from .models.jobs import CancelResult, JobEvent, JobStatus, JobTrace, WorkerActivity, WorkersLive
|
|
68
68
|
|
|
69
69
|
# ------------------- Pipelines models ------------------- #
|
|
70
70
|
from .models.pipelines import (
|
|
@@ -80,6 +80,15 @@ from .models.pipelines import (
|
|
|
80
80
|
# ------------------- Search models ------------------- #
|
|
81
81
|
from .models.search import BlockLocation, SearchHit, SearchRequest, SearchResponse, SearchTarget
|
|
82
82
|
|
|
83
|
+
# ------------------- Storage models ------------------- #
|
|
84
|
+
from .models.storage import (
|
|
85
|
+
CollectionStorageResponse,
|
|
86
|
+
DocumentStorageModel,
|
|
87
|
+
PostgresFootprintModel,
|
|
88
|
+
QdrantFootprintModel,
|
|
89
|
+
S3FootprintModel,
|
|
90
|
+
)
|
|
91
|
+
|
|
83
92
|
# ------------------- Public API ------------------- #
|
|
84
93
|
__all__ = [
|
|
85
94
|
"__version__",
|
|
@@ -140,6 +149,7 @@ __all__ = [
|
|
|
140
149
|
"JobTrace",
|
|
141
150
|
"WorkerActivity",
|
|
142
151
|
"WorkersLive",
|
|
152
|
+
"CancelResult",
|
|
143
153
|
# Blobs
|
|
144
154
|
"BlobContent",
|
|
145
155
|
# Pipelines
|
|
@@ -150,6 +160,12 @@ __all__ = [
|
|
|
150
160
|
"EditResponse",
|
|
151
161
|
"StageViewResponse",
|
|
152
162
|
"StageApplyResponse",
|
|
163
|
+
# Storage
|
|
164
|
+
"S3FootprintModel",
|
|
165
|
+
"PostgresFootprintModel",
|
|
166
|
+
"QdrantFootprintModel",
|
|
167
|
+
"DocumentStorageModel",
|
|
168
|
+
"CollectionStorageResponse",
|
|
153
169
|
# Exceptions
|
|
154
170
|
"DocForgeError",
|
|
155
171
|
"APIConnectionError",
|
|
@@ -48,7 +48,7 @@ from .health import HealthStatus
|
|
|
48
48
|
from .ir import DocumentIRModel, IRBlock, IREnrichment, IRFigure, IRTable
|
|
49
49
|
|
|
50
50
|
# ------------------- Jobs models ------------------- #
|
|
51
|
-
from .jobs import JobEvent, JobStatus, JobTrace, WorkerActivity, WorkersLive
|
|
51
|
+
from .jobs import CancelResult, JobEvent, JobStatus, JobTrace, WorkerActivity, WorkersLive
|
|
52
52
|
|
|
53
53
|
# ------------------- Pipelines models ------------------- #
|
|
54
54
|
from .pipelines import (
|
|
@@ -64,6 +64,15 @@ from .pipelines import (
|
|
|
64
64
|
# ------------------- Search models ------------------- #
|
|
65
65
|
from .search import BlockLocation, SearchHit, SearchRequest, SearchResponse, SearchTarget
|
|
66
66
|
|
|
67
|
+
# ------------------- Storage models ------------------- #
|
|
68
|
+
from .storage import (
|
|
69
|
+
CollectionStorageResponse,
|
|
70
|
+
DocumentStorageModel,
|
|
71
|
+
PostgresFootprintModel,
|
|
72
|
+
QdrantFootprintModel,
|
|
73
|
+
S3FootprintModel,
|
|
74
|
+
)
|
|
75
|
+
|
|
67
76
|
# ------------------- Public API ------------------- #
|
|
68
77
|
__all__ = [
|
|
69
78
|
# Shared vocabulary
|
|
@@ -120,6 +129,7 @@ __all__ = [
|
|
|
120
129
|
"JobTrace",
|
|
121
130
|
"WorkerActivity",
|
|
122
131
|
"WorkersLive",
|
|
132
|
+
"CancelResult",
|
|
123
133
|
# Blobs
|
|
124
134
|
"BlobContent",
|
|
125
135
|
# Pipelines
|
|
@@ -130,4 +140,10 @@ __all__ = [
|
|
|
130
140
|
"EditResponse",
|
|
131
141
|
"StageViewResponse",
|
|
132
142
|
"StageApplyResponse",
|
|
143
|
+
# Storage
|
|
144
|
+
"S3FootprintModel",
|
|
145
|
+
"PostgresFootprintModel",
|
|
146
|
+
"QdrantFootprintModel",
|
|
147
|
+
"DocumentStorageModel",
|
|
148
|
+
"CollectionStorageResponse",
|
|
133
149
|
]
|
|
@@ -17,8 +17,14 @@ class JobStatus(BaseModel):
|
|
|
17
17
|
Attributes:
|
|
18
18
|
job_id (str): The job row's UUID.
|
|
19
19
|
document_id (str): The document being ingested.
|
|
20
|
+
document_filename (str | None): The document's filename, joined at read (None if the
|
|
21
|
+
document is gone).
|
|
20
22
|
collection_id (str): Its collection.
|
|
21
|
-
|
|
23
|
+
collection_name (str | None): The collection's name, joined at read (None if the
|
|
24
|
+
collection is gone).
|
|
25
|
+
status (str): queued / running / done / failed / cancelled.
|
|
26
|
+
cancel_requested (bool): A cooperative stop has been requested; the running job stops at
|
|
27
|
+
its next stage boundary (still 'running' until it does).
|
|
22
28
|
progress (int): 0–100 (completed pipeline nodes over total).
|
|
23
29
|
current_stage (str | None): The node currently (or last) executed.
|
|
24
30
|
error (str | None): The failure, verbatim — only set when status is failed.
|
|
@@ -35,8 +41,21 @@ class JobStatus(BaseModel):
|
|
|
35
41
|
|
|
36
42
|
job_id: str = Field(description="The job row's UUID.")
|
|
37
43
|
document_id: str = Field(description="The document being ingested.")
|
|
44
|
+
document_filename: str | None = Field(
|
|
45
|
+
default=None,
|
|
46
|
+
description="The document's filename, joined at read (None if the document is gone).",
|
|
47
|
+
)
|
|
38
48
|
collection_id: str = Field(description="Its collection.")
|
|
39
|
-
|
|
49
|
+
collection_name: str | None = Field(
|
|
50
|
+
default=None,
|
|
51
|
+
description="The collection's name, joined at read (None if the collection is gone).",
|
|
52
|
+
)
|
|
53
|
+
status: str = Field(description="queued / running / done / failed / cancelled.")
|
|
54
|
+
cancel_requested: bool = Field(
|
|
55
|
+
default=False,
|
|
56
|
+
description="A cooperative stop has been requested; the running job stops at its next "
|
|
57
|
+
"stage boundary (still 'running' until it does).",
|
|
58
|
+
)
|
|
40
59
|
progress: int = Field(description="0-100, completed pipeline nodes over total.")
|
|
41
60
|
current_stage: str | None = Field(default=None, description="Node currently/last executed.")
|
|
42
61
|
error: str | None = Field(default=None, description="Failure detail when status=failed.")
|
|
@@ -130,6 +149,8 @@ class WorkerActivity(BaseModel):
|
|
|
130
149
|
|
|
131
150
|
Attributes:
|
|
132
151
|
worker_id (str): The worker's stable id (its hostname).
|
|
152
|
+
worker_name (str | None): Its friendly display name (WORKER_NAME, defaults to the
|
|
153
|
+
hostname); None only for a heartbeat row written before this column landed.
|
|
133
154
|
alive (bool): Its heartbeat is fresher than the liveness threshold.
|
|
134
155
|
busy (bool): It currently owns at least one RUNNING job.
|
|
135
156
|
last_seen (datetime | None): Its last heartbeat tick (None when no heartbeat row exists).
|
|
@@ -138,6 +159,11 @@ class WorkerActivity(BaseModel):
|
|
|
138
159
|
"""
|
|
139
160
|
|
|
140
161
|
worker_id: str = Field(description="The worker's stable id (its hostname).")
|
|
162
|
+
worker_name: str | None = Field(
|
|
163
|
+
default=None,
|
|
164
|
+
description="Friendly display name (WORKER_NAME, defaults to hostname); None for a "
|
|
165
|
+
"pre-column row.",
|
|
166
|
+
)
|
|
141
167
|
alive: bool = Field(description="Heartbeat fresher than the liveness threshold.")
|
|
142
168
|
busy: bool = Field(description="Owns at least one RUNNING job right now.")
|
|
143
169
|
last_seen: datetime | None = Field(
|
|
@@ -162,4 +188,38 @@ class WorkersLive(BaseModel):
|
|
|
162
188
|
)
|
|
163
189
|
|
|
164
190
|
|
|
165
|
-
|
|
191
|
+
class CancelResult(BaseModel):
|
|
192
|
+
"""
|
|
193
|
+
The typed outcome of a cancel request — what the job's state is after the call.
|
|
194
|
+
|
|
195
|
+
Attributes:
|
|
196
|
+
job_id (str): The targeted job's UUID.
|
|
197
|
+
status (str): The job's status AFTER the call (cancelled / running).
|
|
198
|
+
cancel_requested (bool): Whether a cooperative stop is now pending (true only while a
|
|
199
|
+
running job is still winding down to the CANCELLED terminal state).
|
|
200
|
+
outcome (str): cancelled (now terminal) | cancellation_requested (running, will stop at
|
|
201
|
+
the next stage boundary).
|
|
202
|
+
detail (str): A human-readable description of what happened.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
job_id: str = Field(description="The targeted job's UUID.")
|
|
206
|
+
status: str = Field(description="The job's status AFTER the call (cancelled / running).")
|
|
207
|
+
cancel_requested: bool = Field(
|
|
208
|
+
description="Whether a cooperative stop is now pending (true only while a running job is "
|
|
209
|
+
"still winding down to the CANCELLED terminal state)."
|
|
210
|
+
)
|
|
211
|
+
outcome: str = Field(
|
|
212
|
+
description="cancelled (now terminal) | cancellation_requested (running, will stop at "
|
|
213
|
+
"the next stage boundary)."
|
|
214
|
+
)
|
|
215
|
+
detail: str = Field(description="A human-readable description of what happened.")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
__all__ = [
|
|
219
|
+
"JobStatus",
|
|
220
|
+
"JobEvent",
|
|
221
|
+
"JobTrace",
|
|
222
|
+
"WorkerActivity",
|
|
223
|
+
"WorkersLive",
|
|
224
|
+
"CancelResult",
|
|
225
|
+
]
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# ====== Code Summary ======
|
|
2
|
+
# Response models for the collection storage-footprint endpoint, mirrored field-for-field from the
|
|
3
|
+
# DocForge backend router models (S3FootprintModel / PostgresFootprintModel / QdrantFootprintModel /
|
|
4
|
+
# DocumentStorageModel / CollectionStorageResponse). S3 bytes are EXACT; Postgres and Qdrant bytes are
|
|
5
|
+
# ESTIMATES — each section carries its own ``estimated`` flag.
|
|
6
|
+
|
|
7
|
+
# ====== Third-Party Library Imports ======
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class S3FootprintModel(BaseModel):
|
|
12
|
+
"""
|
|
13
|
+
EXACT S3 bytes from the content-addressed blob registry (``estimated`` is always false).
|
|
14
|
+
|
|
15
|
+
Attributes:
|
|
16
|
+
original_bytes (int): Uploaded source file bytes.
|
|
17
|
+
rendered_bytes (int): Derived-blob bytes (canonical PDF, page renders, crops).
|
|
18
|
+
total_bytes (int): Logical bytes (original + rendered).
|
|
19
|
+
physical_unique_bytes (int): Deduped disk cost — a blob shared across documents counts
|
|
20
|
+
once (<= total).
|
|
21
|
+
estimated (bool): Always false: S3 bytes are measured exactly.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
original_bytes: int = Field(description="Uploaded source file bytes.")
|
|
25
|
+
rendered_bytes: int = Field(
|
|
26
|
+
description="Derived-blob bytes (canonical PDF, page renders, crops)."
|
|
27
|
+
)
|
|
28
|
+
total_bytes: int = Field(description="Logical bytes (original + rendered).")
|
|
29
|
+
physical_unique_bytes: int = Field(
|
|
30
|
+
description="Deduped disk cost — a blob shared across documents counts once (<= total)."
|
|
31
|
+
)
|
|
32
|
+
estimated: bool = Field(description="Always false: S3 bytes are measured exactly.")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class PostgresFootprintModel(BaseModel):
|
|
36
|
+
"""
|
|
37
|
+
ESTIMATED Postgres row bytes via ``pg_column_size`` (excludes index/TOAST/bloat).
|
|
38
|
+
|
|
39
|
+
Attributes:
|
|
40
|
+
documents_bytes (int): ``document`` + ``page`` rows.
|
|
41
|
+
ir_blocks_bytes (int): ``block`` + ``block_table`` + ``block_figure`` rows.
|
|
42
|
+
enrichment_bytes (int): ``block_enrichment`` + ``enrichment_attempt`` rows.
|
|
43
|
+
chunks_bytes (int): ``chunk`` + ``chunk_block`` + ``chunk_metadata`` + ``entity_mention`` rows.
|
|
44
|
+
metadata_bytes (int): ``document_metadata`` rows.
|
|
45
|
+
observability_bytes (int): ``job`` + ``job_stage_event`` rows.
|
|
46
|
+
total_bytes (int): Sum of every bucket.
|
|
47
|
+
estimated (bool): Always true: real row bytes, no index/TOAST/bloat.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
documents_bytes: int = Field(description="``document`` + ``page`` rows.")
|
|
51
|
+
ir_blocks_bytes: int = Field(description="``block`` + ``block_table`` + ``block_figure`` rows.")
|
|
52
|
+
enrichment_bytes: int = Field(description="``block_enrichment`` + ``enrichment_attempt`` rows.")
|
|
53
|
+
chunks_bytes: int = Field(
|
|
54
|
+
description="``chunk`` + ``chunk_block`` + ``chunk_metadata`` + ``entity_mention`` rows."
|
|
55
|
+
)
|
|
56
|
+
metadata_bytes: int = Field(description="``document_metadata`` rows.")
|
|
57
|
+
observability_bytes: int = Field(description="``job`` + ``job_stage_event`` rows.")
|
|
58
|
+
total_bytes: int = Field(description="Sum of every bucket.")
|
|
59
|
+
estimated: bool = Field(description="Always true: real row bytes, no index/TOAST/bloat.")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class QdrantFootprintModel(BaseModel):
|
|
63
|
+
"""
|
|
64
|
+
ESTIMATED vector-store bytes (points x declared shape — excludes HNSW index overhead).
|
|
65
|
+
|
|
66
|
+
Attributes:
|
|
67
|
+
points (int): Point count (collection total, or a document's points).
|
|
68
|
+
dense_bytes (int): On-disk float32 dense bytes, summed per named vector weighted by its
|
|
69
|
+
carrier count. Excludes the int8 quantized RAM-resident copy — this is disk, not RAM.
|
|
70
|
+
sparse_bytes (int): ``points * avg_sparse_entries * 8`` (int32 index + float32 value).
|
|
71
|
+
payload_bytes (int): ``points * avg_payload_json_bytes``.
|
|
72
|
+
total_bytes (int): Sum of dense + sparse + payload.
|
|
73
|
+
estimated (bool): Always true: count-based, excludes index overhead.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
points: int = Field(description="Point count (collection total, or a document's points).")
|
|
77
|
+
dense_bytes: int = Field(
|
|
78
|
+
description=(
|
|
79
|
+
"On-disk float32 dense bytes, summed per named vector weighted by its carrier count "
|
|
80
|
+
"(content_dense on every point, each meta vector only on its field's documents). "
|
|
81
|
+
"Excludes the int8 quantized RAM-resident copy — this is disk, not RAM."
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
sparse_bytes: int = Field(
|
|
85
|
+
description="``points x avg_sparse_entries x 8`` (int32 index + float32 value)."
|
|
86
|
+
)
|
|
87
|
+
payload_bytes: int = Field(description="``points x avg_payload_json_bytes``.")
|
|
88
|
+
total_bytes: int = Field(description="Sum of dense + sparse + payload.")
|
|
89
|
+
estimated: bool = Field(description="Always true: count-based, excludes index overhead.")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DocumentStorageModel(BaseModel):
|
|
93
|
+
"""
|
|
94
|
+
One document's footprint across the three stores.
|
|
95
|
+
|
|
96
|
+
Attributes:
|
|
97
|
+
document_id (str): The document's UUID.
|
|
98
|
+
filename (str): The document's display name.
|
|
99
|
+
s3 (S3FootprintModel): EXACT S3 bytes.
|
|
100
|
+
postgres (PostgresFootprintModel): ESTIMATED Postgres row bytes.
|
|
101
|
+
qdrant (QdrantFootprintModel): ESTIMATED vector-store bytes.
|
|
102
|
+
total_bytes (int): S3 (logical) + Postgres + Qdrant.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
document_id: str = Field(description="The document's UUID.")
|
|
106
|
+
filename: str = Field(description="The document's display name.")
|
|
107
|
+
s3: S3FootprintModel = Field(description="EXACT S3 bytes.")
|
|
108
|
+
postgres: PostgresFootprintModel = Field(description="ESTIMATED Postgres row bytes.")
|
|
109
|
+
qdrant: QdrantFootprintModel = Field(description="ESTIMATED vector-store bytes.")
|
|
110
|
+
total_bytes: int = Field(description="S3 (logical) + Postgres + Qdrant.")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class CollectionStorageResponse(BaseModel):
|
|
114
|
+
"""
|
|
115
|
+
A collection's material footprint per store, plus the per-document breakdown (heaviest first).
|
|
116
|
+
|
|
117
|
+
S3 bytes are EXACT; Postgres and Qdrant bytes are ESTIMATES (each section flags this via its own
|
|
118
|
+
``estimated``). ``grand_total_bytes`` uses the DEDUPED S3 disk cost (``physical_unique_bytes``),
|
|
119
|
+
so it reflects real hardware rather than the logical per-document sum.
|
|
120
|
+
|
|
121
|
+
Attributes:
|
|
122
|
+
collection_id (str): The measured collection's UUID.
|
|
123
|
+
s3 (S3FootprintModel): EXACT S3 totals (logical + deduped physical).
|
|
124
|
+
postgres (PostgresFootprintModel): ESTIMATED Postgres row bytes.
|
|
125
|
+
qdrant (QdrantFootprintModel): ESTIMATED vector-store bytes.
|
|
126
|
+
grand_total_bytes (int): Material footprint — S3 physical_unique + Postgres + Qdrant.
|
|
127
|
+
documents (list[DocumentStorageModel]): Per-document breakdown, sorted by total bytes
|
|
128
|
+
descending (doubles as top-N).
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
collection_id: str = Field(description="The measured collection's UUID.")
|
|
132
|
+
s3: S3FootprintModel = Field(description="EXACT S3 totals (logical + deduped physical).")
|
|
133
|
+
postgres: PostgresFootprintModel = Field(description="ESTIMATED Postgres row bytes.")
|
|
134
|
+
qdrant: QdrantFootprintModel = Field(description="ESTIMATED vector-store bytes.")
|
|
135
|
+
grand_total_bytes: int = Field(
|
|
136
|
+
description="Material footprint — S3 physical_unique + Postgres + Qdrant."
|
|
137
|
+
)
|
|
138
|
+
documents: list[DocumentStorageModel] = Field(
|
|
139
|
+
description="Per-document breakdown, sorted by total bytes descending (doubles as top-N)."
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
__all__ = [
|
|
144
|
+
"S3FootprintModel",
|
|
145
|
+
"PostgresFootprintModel",
|
|
146
|
+
"QdrantFootprintModel",
|
|
147
|
+
"DocumentStorageModel",
|
|
148
|
+
"CollectionStorageResponse",
|
|
149
|
+
]
|
|
@@ -10,6 +10,7 @@ from ..models.collections import (
|
|
|
10
10
|
CreateCollectionRequest,
|
|
11
11
|
UpdateCollectionRequest,
|
|
12
12
|
)
|
|
13
|
+
from ..models.storage import CollectionStorageResponse
|
|
13
14
|
from ._base import AsyncResource, SyncResource, _ResourceMixin
|
|
14
15
|
|
|
15
16
|
|
|
@@ -80,6 +81,18 @@ class _CollectionsSpecs(_ResourceMixin):
|
|
|
80
81
|
"""
|
|
81
82
|
return RequestSpec("DELETE", f"{self._COLLECTIONS_PATH}/{collection_id}")
|
|
82
83
|
|
|
84
|
+
def _storage_spec(self, collection_id: str) -> RequestSpec:
|
|
85
|
+
"""
|
|
86
|
+
Build the spec for measuring a collection's material storage footprint.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
collection_id (str): The collection's UUID.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
RequestSpec: A GET on the collection's storage sub-resource.
|
|
93
|
+
"""
|
|
94
|
+
return RequestSpec("GET", f"{self._COLLECTIONS_PATH}/{collection_id}/storage")
|
|
95
|
+
|
|
83
96
|
|
|
84
97
|
class AsyncCollections(AsyncResource, _CollectionsSpecs):
|
|
85
98
|
"""Asynchronous collection management (list / get / create / update / delete)."""
|
|
@@ -141,6 +154,20 @@ class AsyncCollections(AsyncResource, _CollectionsSpecs):
|
|
|
141
154
|
"""
|
|
142
155
|
return await self._transport.request(self._delete_spec(collection_id), type(None))
|
|
143
156
|
|
|
157
|
+
async def storage(self, collection_id: str) -> CollectionStorageResponse:
|
|
158
|
+
"""
|
|
159
|
+
Measure a collection's material footprint per store (S3 exact, Postgres/Qdrant estimated).
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
collection_id (str): The collection's UUID.
|
|
163
|
+
|
|
164
|
+
Returns:
|
|
165
|
+
CollectionStorageResponse: Per-store totals + the per-document breakdown, heaviest first.
|
|
166
|
+
"""
|
|
167
|
+
return await self._transport.request(
|
|
168
|
+
self._storage_spec(collection_id), CollectionStorageResponse
|
|
169
|
+
)
|
|
170
|
+
|
|
144
171
|
|
|
145
172
|
class SyncCollections(SyncResource, _CollectionsSpecs):
|
|
146
173
|
"""Synchronous collection management (list / get / create / update / delete)."""
|
|
@@ -200,5 +227,17 @@ class SyncCollections(SyncResource, _CollectionsSpecs):
|
|
|
200
227
|
"""
|
|
201
228
|
return self._transport.request(self._delete_spec(collection_id), type(None))
|
|
202
229
|
|
|
230
|
+
def storage(self, collection_id: str) -> CollectionStorageResponse:
|
|
231
|
+
"""
|
|
232
|
+
Measure a collection's material footprint per store (S3 exact, Postgres/Qdrant estimated).
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
collection_id (str): The collection's UUID.
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
CollectionStorageResponse: Per-store totals + the per-document breakdown, heaviest first.
|
|
239
|
+
"""
|
|
240
|
+
return self._transport.request(self._storage_spec(collection_id), CollectionStorageResponse)
|
|
241
|
+
|
|
203
242
|
|
|
204
243
|
__all__ = ["AsyncCollections", "SyncCollections"]
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
# ====== Local Project Imports ======
|
|
7
7
|
from .._requestspec import RequestSpec
|
|
8
|
-
from ..models.jobs import JobStatus, JobTrace, WorkersLive
|
|
8
|
+
from ..models.jobs import CancelResult, JobStatus, JobTrace, WorkersLive
|
|
9
9
|
from ._base import AsyncResource, SyncResource, _ResourceMixin
|
|
10
10
|
|
|
11
11
|
|
|
@@ -59,6 +59,20 @@ class _JobsSpecs(_ResourceMixin):
|
|
|
59
59
|
"""
|
|
60
60
|
return RequestSpec("GET", f"{self._JOBS_PATH}/workers/live")
|
|
61
61
|
|
|
62
|
+
def _cancel_spec(self, job_id: str, force: bool) -> RequestSpec:
|
|
63
|
+
"""
|
|
64
|
+
Build the spec for cancelling a job.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
job_id (str): The job's UUID.
|
|
68
|
+
force (bool): Immediately terminate a running/wedged job instead of asking it to stop
|
|
69
|
+
cooperatively at its next stage boundary.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
RequestSpec: A POST on the job's ``/cancel`` route carrying ``force`` as a query param.
|
|
73
|
+
"""
|
|
74
|
+
return RequestSpec("POST", f"{self._JOBS_PATH}/{job_id}/cancel", params={"force": force})
|
|
75
|
+
|
|
62
76
|
|
|
63
77
|
class AsyncJobs(AsyncResource, _JobsSpecs):
|
|
64
78
|
"""Asynchronous ingestion-job monitoring."""
|
|
@@ -108,6 +122,22 @@ class AsyncJobs(AsyncResource, _JobsSpecs):
|
|
|
108
122
|
"""
|
|
109
123
|
return await self._transport.request(self._live_workers_spec(), WorkersLive)
|
|
110
124
|
|
|
125
|
+
async def cancel(self, job_id: str, force: bool = False) -> CancelResult:
|
|
126
|
+
"""
|
|
127
|
+
Cancel an ingestion job — cooperatively for a running job, immediately for a queued or
|
|
128
|
+
wedged one.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
job_id (str): The job's UUID.
|
|
132
|
+
force (bool): Immediately terminate a running/wedged job regardless of worker state
|
|
133
|
+
instead of asking it to stop cooperatively at its next stage boundary.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
CancelResult: The job's post-call status, whether a cooperative stop is pending, and
|
|
137
|
+
the outcome.
|
|
138
|
+
"""
|
|
139
|
+
return await self._transport.request(self._cancel_spec(job_id, force), CancelResult)
|
|
140
|
+
|
|
111
141
|
|
|
112
142
|
class SyncJobs(SyncResource, _JobsSpecs):
|
|
113
143
|
"""Synchronous ingestion-job monitoring."""
|
|
@@ -157,5 +187,21 @@ class SyncJobs(SyncResource, _JobsSpecs):
|
|
|
157
187
|
"""
|
|
158
188
|
return self._transport.request(self._live_workers_spec(), WorkersLive)
|
|
159
189
|
|
|
190
|
+
def cancel(self, job_id: str, force: bool = False) -> CancelResult:
|
|
191
|
+
"""
|
|
192
|
+
Cancel an ingestion job — cooperatively for a running job, immediately for a queued or
|
|
193
|
+
wedged one.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
job_id (str): The job's UUID.
|
|
197
|
+
force (bool): Immediately terminate a running/wedged job regardless of worker state
|
|
198
|
+
instead of asking it to stop cooperatively at its next stage boundary.
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
CancelResult: The job's post-call status, whether a cooperative stop is pending, and
|
|
202
|
+
the outcome.
|
|
203
|
+
"""
|
|
204
|
+
return self._transport.request(self._cancel_spec(job_id, force), CancelResult)
|
|
205
|
+
|
|
160
206
|
|
|
161
207
|
__all__ = ["AsyncJobs", "SyncJobs"]
|