dataerai-sdk 0.3.0b8__tar.gz → 0.3.0.dev1__tar.gz

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataerai-sdk
-Version: 0.3.0b8
+Version: 0.3.0.dev1
 Summary: Python SDK for the Dataerai transfer daemon
 Author: Dataerai
 License: DATAERAI PROPRIETARY SOFTWARE LICENSE AGREEMENT
@@ -367,6 +367,16 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Provides-Extra: chat-capture
+Requires-Dist: httpx>=0.27; extra == "chat-capture"
+Requires-Dist: cryptography>=42; extra == "chat-capture"
+Provides-Extra: instruments
+Requires-Dist: numpy>=1.26; extra == "instruments"
+Requires-Dist: h5py>=3.7; extra == "instruments"
+Requires-Dist: netCDF4>=1.6; extra == "instruments"
+Requires-Dist: dataerai-sdk[rest]; extra == "instruments"
+Provides-Extra: prepare
+Requires-Dist: pyarrow>=25.0.1; extra == "prepare"
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
 Requires-Dist: pytest-mock>=3; extra == "dev"
@@ -375,6 +385,9 @@ Requires-Dist: pyyaml>=6; extra == "dev"
 Provides-Extra: rest
 Requires-Dist: httpx>=0.27; extra == "rest"
 Requires-Dist: keyring>=25; extra == "rest"
+Provides-Extra: ci
+Requires-Dist: httpx>=0.27; extra == "ci"
+Requires-Dist: requests>=2.31; extra == "ci"
 Provides-Extra: envelope
 Requires-Dist: cryptography>=42; extra == "envelope"
 Requires-Dist: blake3>=0.3; extra == "envelope"
@@ -390,7 +403,7 @@ Requires-Dist: psutil>=5.9; extra == "nn-tensorflow"
 Requires-Dist: nvidia-ml-py>=12; extra == "nn-tensorflow"
 Provides-Extra: nn-pytorch
 Requires-Dist: ipython>=8; extra == "nn-pytorch"
-Requires-Dist: torch>=2.6; extra == "nn-pytorch"
+Requires-Dist: torch>=2.10; extra == "nn-pytorch"
 Requires-Dist: numpy>=1.26; extra == "nn-pytorch"
 Requires-Dist: psutil>=5.9; extra == "nn-pytorch"
 Requires-Dist: nvidia-ml-py>=12; extra == "nn-pytorch"
@@ -405,9 +418,16 @@ Requires-Dist: requests>=2.31; extra == "ml-io"
 Requires-Dist: s3fs>=2024.0; extra == "ml-io"
 Provides-Extra: ml
 Requires-Dist: dataerai-sdk[ml-io]; extra == "ml"
-Requires-Dist: torch>=2.4; extra == "ml"
+Requires-Dist: torch>=2.10; extra == "ml"
 Requires-Dist: torchdata>=0.9; extra == "ml"
 Requires-Dist: safetensors>=0.4; extra == "ml"
+Provides-Extra: ml-lightning
+Requires-Dist: dataerai-sdk[ml]; extra == "ml-lightning"
+Requires-Dist: lightning<3,>=2.4; extra == "ml-lightning"
+Provides-Extra: ml-transformers
+Requires-Dist: dataerai-sdk[ml]; extra == "ml-transformers"
+Requires-Dist: transformers<6,>=4.46; extra == "ml-transformers"
+Requires-Dist: accelerate<2,>=1.0; extra == "ml-transformers"
 Provides-Extra: croissant
 Requires-Dist: mlcroissant>=1.0; extra == "croissant"
 Provides-Extra: ml-gpu-cu12
@@ -448,6 +468,7 @@ Requires-Dist: torch; extra == "ml-stream-train"
 Provides-Extra: runner
 Requires-Dist: requests>=2; extra == "runner"
 Requires-Dist: nvidia-ml-py>=12.535; extra == "runner"
+Requires-Dist: psutil>=5.9; extra == "runner"
 Provides-Extra: metaextract
 Requires-Dist: numpy>=1.23; extra == "metaextract"
 Requires-Dist: h5py>=3.7; extra == "metaextract"
@@ -513,7 +534,7 @@ resumable transfers; the optional REST client exposes every public API path.
 ## Installation
 
 ```bash
-pip install --pre "dataerai-sdk>=0.2.0b1,<0.3"
+pip install --pre "dataerai-sdk>=0.3.0b1,<0.4"
 # or, from source:
 pip install -e sdk/python/
 ```
@@ -521,13 +542,13 @@ pip install -e sdk/python/
 Optional ML adapters are installed separately so the base SDK stays small:
 
 ```bash
-pip install --pre "dataerai-sdk[rest]>=0.2.0b1,<0.3"        # Full public REST API
-pip install --pre "dataerai-sdk[ml]>=0.2.0b1,<0.3"          # PyTorch Zarr datasets
-pip install --pre "dataerai-sdk[ml-keras]>=0.2.0b1,<0.3"    # Keras batches
-pip install --pre "dataerai-sdk[ml-hls4ml]>=0.2.0b1,<0.3"   # HLS4ML conversion
-pip install --pre "dataerai-sdk[notebook]>=0.2.0b1,<0.3"    # IPython %dataerai magic
-pip install --pre "dataerai-sdk[nn-tensorflow]>=0.2.0b1,<0.3" # TensorFlow provenance
-pip install --pre "dataerai-sdk[envelope]>=0.2.0b1,<0.3"    # Encrypted-envelope reader
+pip install --pre "dataerai-sdk[rest]>=0.3.0b1,<0.4"        # Full public REST API
+pip install --pre "dataerai-sdk[ml]>=0.3.0b1,<0.4"          # PyTorch Zarr datasets
+pip install --pre "dataerai-sdk[ml-keras]>=0.3.0b1,<0.4"    # Keras batches
+pip install --pre "dataerai-sdk[ml-hls4ml]>=0.3.0b1,<0.4"   # HLS4ML conversion
+pip install --pre "dataerai-sdk[notebook]>=0.3.0b1,<0.4"    # IPython %dataerai magic
+pip install --pre "dataerai-sdk[nn-tensorflow]>=0.3.0b1,<0.4" # TensorFlow provenance
+pip install --pre "dataerai-sdk[envelope]>=0.3.0b1,<0.4"    # Encrypted-envelope reader
 ```
 
 ## Full public REST API
@@ -557,6 +578,24 @@ the configured Dataerai server. For headless jobs, set `DATAERAI_SERVER` and
 `DATAERAI_TOKEN`, or provide the CLI-compatible JSON credential object through
 `DATAERAI_CREDENTIALS_JSON`.
 
+### Listing helpers
+
+`search_assets(...)` iterates every page of a search. Pass `slim=True` when ids,
+titles, aliases and a file count are all you read: the console then sends its
+minimum browse row (no `metadata`, no content `files`, a 280-character
+`description` preview with `description_truncated`); `find_asset(alias=...)`
+does. `list_collections(owner_type, owner_id, ...)` and
+`list_projects(q=..., scope=...)` return the whole list, as before; pass
+`limit` (1–500 rows per request) or `cursor` to walk the console's keyset pages
+instead, as an iterator like `search_assets` (rows carry description previews,
+and projects come by name rather than newest first).
+
+```python
+with RestClient() as api:
+    for collection in api.list_collections("project", project_id, limit=500):
+        print(collection["title"])
+```
+
 ### Verifiable credentials
 
 The REST client provides typed convenience methods without accepting authority
@@ -763,6 +802,23 @@ If a traced cell fails, its source, outputs, error, and traceback are published
 immediately; an explicit `%dataerai --finish` is not required for that failed
 run.
 
+The log asset also carries a bounded `notebook_execution` summary in its own,
+searchable metadata: run identity and timing, the destination, the notebook's
+filename and checksum, the Python version and package count, cell/error/product
+counts with `failed_cell_indexes`, and for each of the first 200 cells its
+`index`, `execution_count`, `duration_seconds`, `ok`, and `error_type`. The
+block stays at or below 64 KiB; `cells_truncated` is always present: `false` for a
+complete cell list and `true` for a shortened one.
+It never contains cell source, output, error messages, or tracebacks: those
+stay in the log file, because metadata can be readable by people who cannot
+open the file. Any `.ipynb` file uploaded through the traced session also
+gains a `notebook` summary, even without `--notebook` (`nbformat`,
+`cell_count`, `code_cell_count`, `kernel`, `language`, `has_outputs`, marked with
+`schema: dataerai.notebook-summary.v1` and the `sha256` of the file it
+describes). The SDK only writes or replaces a `notebook` key it wrote itself, so
+a value you set on the notebook asset is kept. Both are ordinary metadata, so a
+search such as `notebook_execution.summary.error_count > 0` finds failed runs.
+
 Tracing is deliberately opt-in because cell source and output can contain
 secrets or sensitive research data. Review the notebook before enabling it.
 The recorder does not read environment-variable values or scan/upload arbitrary
@@ -834,6 +890,12 @@ final = tracker.save_checkpoint(
 
 ## PyTorch neural-network provenance
 
+For ordered minibatch membership, exact sample exposure counts, optional
+cryptographic snapshot verification and bounded selected-weight updates, see
+[training batch audit](https://docs.dataerai.com/sdks/training-batch-audit).
+The local example is `examples/training_batch_audit.py`; capture is opt-in and
+has finite retention, batch, cardinality and disk limits.
+
 Install the PyTorch adapter when a training run must preserve native model and
 optimizer state:
 
@@ -1105,12 +1167,13 @@ fail at the point of cause — uploads are rejected, and a rejected upload can
 leave a contentless asset that surfaces much later as "no downloadable
 content".
 
-> **`is_full` is necessary, not sufficient.** `True` means an upload will be
-> rejected. `False` does *not* guarantee one succeeds: the console also
-> hard-blocks on billing suspension, enforces a temporary grace ceiling instead
-> of `data_volume_limit` during a grace window, and walks up parent
-> allocations — none of which the daemon reports, so none are visible here.
-> Read `False` as "the reported quota has room".
+> **`is_full` reports quota, not upload admission.** `True` means the reported quota
+> is exhausted — but an open grace window can still admit uploads, since the
+> console then enforces a temporary grace ceiling instead of
+> `data_volume_limit`. `False` does *not* guarantee one succeeds: the console
+> also hard-blocks on billing suspension and walks up parent allocations —
+> none of which the daemon reports, so none are visible here. Read `False` as
+> "the reported quota has room".
 
 A limit of `0` means zero capacity, not unlimited — the console's gate is a
 plain `current + incoming > limit` with no special case for zero. A missing limit
@@ -1140,13 +1203,15 @@ for asset in client.find_collection_assets(collection_id):
 if the server claims another page without a cursor or repeats one.
 
 > **`list_projects()` and `list_collections()` show what you can upload into,
-> not everything you can read.** Both are projections of `list_tree()`, and the
-> daemon omits any project you lack `WRITE_METADATA` on — so a project you can
-> read but not write to is absent, and so are its collections. There is no
-> SDK call that lists read-only projects today.
+> not everything you can read.** Both answer exactly what flattening
+> `list_tree()` would, and the tree omits any project you lack `WRITE_METADATA`
+> on — so a project you can read but not write to is absent, and so are its
+> collections. There is no SDK call that lists read-only projects today. They
+> read bounded keyset pages (`project.list` / `collection.list`) rather than
+> the whole tree, and fall back to `list_tree()` on a daemon without them.
 
 `list_projects()` populates only `project_id`, `name` and `root_collection_id`;
-`tree.list` carries nothing else, so the remaining `Project` fields are `None`.
+the tree carries nothing else, so the remaining `Project` fields are `None`.
 Collections shared with you directly are attached to the personal node, so one
 returned under `owner_type="user"` may carry another project's
 `owner_project_id`.
@@ -1556,6 +1621,9 @@ def on_error(evt):
 def on_sync(evt):
     if evt.skipped_unverified:
         print(f"{evt.skipped_unverified} files skipped — no checksum offered")
+    for failure in evt.failures:
+        entry = failure.entry_id or "sync state"
+        print(f"{entry}: {failure.path}: {failure.reason}")
 ```
 
 The daemon **broadcasts to every connected client**, so handlers also see work
@@ -1616,7 +1684,7 @@ lot with `logging.getLogger("dataerai.events").setLevel(logging.CRITICAL)`.
 | `asset.upload_failed` | `UploadFailedEvent` | |
 | `asset.download_failed` | `DownloadFailedEvent` | |
 | `collection.sync_progress` | `CollectionSyncProgressEvent` | Keyed by `sync_id`, not `transfer_id` |
-| `collection.sync_complete` | `CollectionSyncCompleteEvent` | `skipped_unverified > 0` means files were **not** written |
+| `collection.sync_complete` | `CollectionSyncCompleteEvent` | `failures` lists up to 100 failed paths; `failed` counts all. `skipped_unverified > 0` means files were **not** written |
 
 Names are the daemon's own wire names, so the string here is the string in the
 protocol reference and in the daemon's logs. Porting from the Node SDK, which
@@ -1629,3 +1697,25 @@ spells them `transfer:fileComplete`: drop the camelCase and use `.` for `:`.
 | `DaemonError(code, message)` | Daemon returned a coded error (see `code` attribute) |
 | `DaemonTimeoutError` | Request or transfer exceeded the configured timeout |
 | `ConnectionError` | Daemon disconnected unexpectedly |
+
+## TrainKit framework adapters
+
+Install `dataerai-sdk[ml-lightning]` for `dataerai.ml.lightning.TrainKitCallback`,
+or `dataerai-sdk[ml-transformers]` for `dataerai.ml.huggingface.TrainKitTrainer`.
+`dataerai.ml.integrations.ModelInputCollator` converts image/label sample pairs
+into named model inputs. The generic loader now accepts PyTorch collation,
+last-batch, generator, worker-initialization and multiprocessing controls.
+
+Use one distributed partition owner: TrainKit for torchrun/Lightning;
+Accelerate for Hugging Face/custom Accelerate loops. GDS requires zero worker
+processes and disabled memory pinning. See the [framework recipes](https://docs.dataerai.com/sdks/ml-toolkit#training-framework-integrations)
+for complete examples and checkpoint behavior.
+
+Lightning caches training batch limits. `TrainKitCallback` rejects an epoch whose
+shard reshuffle changes the loader batch count before consuming any batch in that
+epoch. For distributed multi-epoch Lightning, use full-sized shards or a shard
+count divisible by the world size; variable batch-count layouts remain unsupported.
+
+## Prepare scientific tables for random reads
+
+Install `pip install "dataerai-sdk[prepare]"` and use `dataerai-prepare input.csv output.parquet` to create Parquet row groups or Arrow IPC file batches with embedded source provenance. See the [local preparation guide](docs/preparation.md) for explicit schemas, exact decimal conversion, compression, and indexed reader examples.

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/README.md

@@ -14,7 +14,7 @@ resumable transfers; the optional REST client exposes every public API path.
 ## Installation
 
 ```bash
-pip install --pre "dataerai-sdk>=0.2.0b1,<0.3"
+pip install --pre "dataerai-sdk>=0.3.0b1,<0.4"
 # or, from source:
 pip install -e sdk/python/
 ```
@@ -22,13 +22,13 @@ pip install -e sdk/python/
 Optional ML adapters are installed separately so the base SDK stays small:
 
 ```bash
-pip install --pre "dataerai-sdk[rest]>=0.2.0b1,<0.3"        # Full public REST API
-pip install --pre "dataerai-sdk[ml]>=0.2.0b1,<0.3"          # PyTorch Zarr datasets
-pip install --pre "dataerai-sdk[ml-keras]>=0.2.0b1,<0.3"    # Keras batches
-pip install --pre "dataerai-sdk[ml-hls4ml]>=0.2.0b1,<0.3"   # HLS4ML conversion
-pip install --pre "dataerai-sdk[notebook]>=0.2.0b1,<0.3"    # IPython %dataerai magic
-pip install --pre "dataerai-sdk[nn-tensorflow]>=0.2.0b1,<0.3" # TensorFlow provenance
-pip install --pre "dataerai-sdk[envelope]>=0.2.0b1,<0.3"    # Encrypted-envelope reader
+pip install --pre "dataerai-sdk[rest]>=0.3.0b1,<0.4"        # Full public REST API
+pip install --pre "dataerai-sdk[ml]>=0.3.0b1,<0.4"          # PyTorch Zarr datasets
+pip install --pre "dataerai-sdk[ml-keras]>=0.3.0b1,<0.4"    # Keras batches
+pip install --pre "dataerai-sdk[ml-hls4ml]>=0.3.0b1,<0.4"   # HLS4ML conversion
+pip install --pre "dataerai-sdk[notebook]>=0.3.0b1,<0.4"    # IPython %dataerai magic
+pip install --pre "dataerai-sdk[nn-tensorflow]>=0.3.0b1,<0.4" # TensorFlow provenance
+pip install --pre "dataerai-sdk[envelope]>=0.3.0b1,<0.4"    # Encrypted-envelope reader
 ```
 
 ## Full public REST API
@@ -58,6 +58,24 @@ the configured Dataerai server. For headless jobs, set `DATAERAI_SERVER` and
 `DATAERAI_TOKEN`, or provide the CLI-compatible JSON credential object through
 `DATAERAI_CREDENTIALS_JSON`.
 
+### Listing helpers
+
+`search_assets(...)` iterates every page of a search. Pass `slim=True` when ids,
+titles, aliases and a file count are all you read: the console then sends its
+minimum browse row (no `metadata`, no content `files`, a 280-character
+`description` preview with `description_truncated`); `find_asset(alias=...)`
+does. `list_collections(owner_type, owner_id, ...)` and
+`list_projects(q=..., scope=...)` return the whole list, as before; pass
+`limit` (1–500 rows per request) or `cursor` to walk the console's keyset pages
+instead, as an iterator like `search_assets` (rows carry description previews,
+and projects come by name rather than newest first).
+
+```python
+with RestClient() as api:
+    for collection in api.list_collections("project", project_id, limit=500):
+        print(collection["title"])
+```
+
 ### Verifiable credentials
 
 The REST client provides typed convenience methods without accepting authority
@@ -264,6 +282,23 @@ If a traced cell fails, its source, outputs, error, and traceback are published
 immediately; an explicit `%dataerai --finish` is not required for that failed
 run.
 
+The log asset also carries a bounded `notebook_execution` summary in its own,
+searchable metadata: run identity and timing, the destination, the notebook's
+filename and checksum, the Python version and package count, cell/error/product
+counts with `failed_cell_indexes`, and for each of the first 200 cells its
+`index`, `execution_count`, `duration_seconds`, `ok`, and `error_type`. The
+block stays at or below 64 KiB; `cells_truncated` is always present: `false` for a
+complete cell list and `true` for a shortened one.
+It never contains cell source, output, error messages, or tracebacks: those
+stay in the log file, because metadata can be readable by people who cannot
+open the file. Any `.ipynb` file uploaded through the traced session also
+gains a `notebook` summary, even without `--notebook` (`nbformat`,
+`cell_count`, `code_cell_count`, `kernel`, `language`, `has_outputs`, marked with
+`schema: dataerai.notebook-summary.v1` and the `sha256` of the file it
+describes). The SDK only writes or replaces a `notebook` key it wrote itself, so
+a value you set on the notebook asset is kept. Both are ordinary metadata, so a
+search such as `notebook_execution.summary.error_count > 0` finds failed runs.
+
 Tracing is deliberately opt-in because cell source and output can contain
 secrets or sensitive research data. Review the notebook before enabling it.
 The recorder does not read environment-variable values or scan/upload arbitrary
@@ -335,6 +370,12 @@ final = tracker.save_checkpoint(
 
 ## PyTorch neural-network provenance
 
+For ordered minibatch membership, exact sample exposure counts, optional
+cryptographic snapshot verification and bounded selected-weight updates, see
+[training batch audit](https://docs.dataerai.com/sdks/training-batch-audit).
+The local example is `examples/training_batch_audit.py`; capture is opt-in and
+has finite retention, batch, cardinality and disk limits.
+
 Install the PyTorch adapter when a training run must preserve native model and
 optimizer state:
 
@@ -606,12 +647,13 @@ fail at the point of cause — uploads are rejected, and a rejected upload can
 leave a contentless asset that surfaces much later as "no downloadable
 content".
 
-> **`is_full` is necessary, not sufficient.** `True` means an upload will be
-> rejected. `False` does *not* guarantee one succeeds: the console also
-> hard-blocks on billing suspension, enforces a temporary grace ceiling instead
-> of `data_volume_limit` during a grace window, and walks up parent
-> allocations — none of which the daemon reports, so none are visible here.
-> Read `False` as "the reported quota has room".
+> **`is_full` reports quota, not upload admission.** `True` means the reported quota
+> is exhausted — but an open grace window can still admit uploads, since the
+> console then enforces a temporary grace ceiling instead of
+> `data_volume_limit`. `False` does *not* guarantee one succeeds: the console
+> also hard-blocks on billing suspension and walks up parent allocations —
+> none of which the daemon reports, so none are visible here. Read `False` as
+> "the reported quota has room".
 
 A limit of `0` means zero capacity, not unlimited — the console's gate is a
 plain `current + incoming > limit` with no special case for zero. A missing limit
@@ -641,13 +683,15 @@ for asset in client.find_collection_assets(collection_id):
 if the server claims another page without a cursor or repeats one.
 
 > **`list_projects()` and `list_collections()` show what you can upload into,
-> not everything you can read.** Both are projections of `list_tree()`, and the
-> daemon omits any project you lack `WRITE_METADATA` on — so a project you can
-> read but not write to is absent, and so are its collections. There is no
-> SDK call that lists read-only projects today.
+> not everything you can read.** Both answer exactly what flattening
+> `list_tree()` would, and the tree omits any project you lack `WRITE_METADATA`
+> on — so a project you can read but not write to is absent, and so are its
+> collections. There is no SDK call that lists read-only projects today. They
+> read bounded keyset pages (`project.list` / `collection.list`) rather than
+> the whole tree, and fall back to `list_tree()` on a daemon without them.
 
 `list_projects()` populates only `project_id`, `name` and `root_collection_id`;
-`tree.list` carries nothing else, so the remaining `Project` fields are `None`.
+the tree carries nothing else, so the remaining `Project` fields are `None`.
 Collections shared with you directly are attached to the personal node, so one
 returned under `owner_type="user"` may carry another project's
 `owner_project_id`.
@@ -1057,6 +1101,9 @@ def on_error(evt):
 def on_sync(evt):
     if evt.skipped_unverified:
         print(f"{evt.skipped_unverified} files skipped — no checksum offered")
+    for failure in evt.failures:
+        entry = failure.entry_id or "sync state"
+        print(f"{entry}: {failure.path}: {failure.reason}")
 ```
 
 The daemon **broadcasts to every connected client**, so handlers also see work
@@ -1117,7 +1164,7 @@ lot with `logging.getLogger("dataerai.events").setLevel(logging.CRITICAL)`.
 | `asset.upload_failed` | `UploadFailedEvent` | |
 | `asset.download_failed` | `DownloadFailedEvent` | |
 | `collection.sync_progress` | `CollectionSyncProgressEvent` | Keyed by `sync_id`, not `transfer_id` |
-| `collection.sync_complete` | `CollectionSyncCompleteEvent` | `skipped_unverified > 0` means files were **not** written |
+| `collection.sync_complete` | `CollectionSyncCompleteEvent` | `failures` lists up to 100 failed paths; `failed` counts all. `skipped_unverified > 0` means files were **not** written |
 
 Names are the daemon's own wire names, so the string here is the string in the
 protocol reference and in the daemon's logs. Porting from the Node SDK, which
@@ -1130,3 +1177,25 @@ spells them `transfer:fileComplete`: drop the camelCase and use `.` for `:`.
 | `DaemonError(code, message)` | Daemon returned a coded error (see `code` attribute) |
 | `DaemonTimeoutError` | Request or transfer exceeded the configured timeout |
 | `ConnectionError` | Daemon disconnected unexpectedly |
+
+## TrainKit framework adapters
+
+Install `dataerai-sdk[ml-lightning]` for `dataerai.ml.lightning.TrainKitCallback`,
+or `dataerai-sdk[ml-transformers]` for `dataerai.ml.huggingface.TrainKitTrainer`.
+`dataerai.ml.integrations.ModelInputCollator` converts image/label sample pairs
+into named model inputs. The generic loader now accepts PyTorch collation,
+last-batch, generator, worker-initialization and multiprocessing controls.
+
+Use one distributed partition owner: TrainKit for torchrun/Lightning;
+Accelerate for Hugging Face/custom Accelerate loops. GDS requires zero worker
+processes and disabled memory pinning. See the [framework recipes](https://docs.dataerai.com/sdks/ml-toolkit#training-framework-integrations)
+for complete examples and checkpoint behavior.
+
+Lightning caches training batch limits. `TrainKitCallback` rejects an epoch whose
+shard reshuffle changes the loader batch count before consuming any batch in that
+epoch. For distributed multi-epoch Lightning, use full-sized shards or a shard
+count divisible by the world size; variable batch-count layouts remain unsupported.
+
+## Prepare scientific tables for random reads
+
+Install `pip install "dataerai-sdk[prepare]"` and use `dataerai-prepare input.csv output.parquet` to create Parquet row groups or Arrow IPC file batches with embedded source provenance. See the [local preparation guide](docs/preparation.md) for explicit schemas, exact decimal conversion, compression, and indexed reader examples.

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/__init__.py

@@ -17,6 +17,7 @@ from ._events import (
     WILDCARD,
     event_name,
     CollectionSyncCompleteEvent,
+    CollectionSyncFailure,
     CollectionSyncProgressEvent,
     DownloadCompleteEvent,
     DownloadFailedEvent,
@@ -120,6 +121,7 @@ __all__ = [
     "DownloadFailedEvent",
     "CollectionSyncProgressEvent",
     "CollectionSyncCompleteEvent",
+    "CollectionSyncFailure",
     "AssetMetadata",
     "NotebookSession",
     "NotebookExecutionTrace",

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/_events.py

@@ -15,7 +15,7 @@ Adding a new event is one edit here — a dataclass plus its row in
 :data:`EVENT_PARSERS` — and the client picks it up with no further changes.
 """
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Callable
 
 from ._types import ProgressEvent
@@ -168,6 +168,18 @@ class CollectionSyncProgressEvent:
     path: str
 
 
+@dataclass
+class CollectionSyncFailure:
+    """One file a collection sync could not write, and why.
+
+    An empty ``entry_id`` is the sync's own state file, which has no entry.
+    """
+
+    entry_id: str
+    path: str
+    reason: str
+
+
 @dataclass
 class CollectionSyncCompleteEvent:
     """A collection sync finished.
@@ -176,6 +188,9 @@ class CollectionSyncCompleteEvent:
     The daemon does **not** write those, so a run with a non-zero count is not
     a clean success however healthy ``downloaded`` looks — this is the shape of
     the "sync only fetched some of my files" reports.
+
+    ``failures`` names the files counted in ``failed`` and why (the first 100;
+    ``failed`` counts all). Empty when the daemon omits the field.
     """
 
     sync_id: str
@@ -187,6 +202,7 @@ class CollectionSyncCompleteEvent:
     pruned: int
     bytes_downloaded: int
     elapsed_ms: int
+    failures: list[CollectionSyncFailure] = field(default_factory=list)
 
 
 # ─── Wire name → parser ───────────────────────────────────────────────────────
@@ -287,6 +303,14 @@ EVENT_PARSERS: dict[str, Callable[[dict], Any]] = {
         pruned=p.get("pruned", 0),
         bytes_downloaded=p.get("bytes_downloaded", 0),
         elapsed_ms=p.get("elapsed_ms", 0),
+        failures=[
+            CollectionSyncFailure(
+                entry_id=f.get("entry_id", ""),
+                path=f.get("path", ""),
+                reason=f.get("reason", ""),
+            )
+            for f in (p.get("failures") or [])
+        ],
     ),
 }
 

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/_framing.py

@@ -18,9 +18,14 @@ DIR_MESSAGE: int = 0x02
 MAX_FRAME_BYTES: int = 32 * 1024 * 1024  # 32 MiB
 
 
+def encode_json(payload: object) -> bytes:
+    """Serialise *payload* as a command frame carries it: compact JSON, UTF-8."""
+    return json.dumps(payload, separators=(",", ":")).encode("utf-8")
+
+
 def encode_frame(payload: dict) -> bytes:
     """Encode *payload* as a command frame (direction = DIR_COMMAND)."""
-    data = json.dumps(payload, separators=(",", ":")).encode("utf-8")
+    data = encode_json(payload)
     if len(data) > MAX_FRAME_BYTES:
         raise ValueError(
             f"payload too large: {len(data):,} bytes (max {MAX_FRAME_BYTES:,})"

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/_types.py

@@ -257,13 +257,15 @@ class Allocation:
         this covers both — a hand-written check almost always tests only bytes,
         and the record cap is the one that tends to bite first.
 
-        **This is necessary, not sufficient.** ``is_full`` is ``True`` means an
-        upload will be rejected. ``False`` does *not* guarantee one will
-        succeed: the console also hard-blocks on billing suspension, enforces a
-        temporary ``grace_bytes`` ceiling instead of ``data_volume_limit``
-        during a grace window, and walks up parent allocations — and the daemon
-        sends none of those, so they are invisible here. Treat a ``False`` as
-        "the reported quota has room", not as a guarantee.
+        **This reports quota, not upload admission.** ``True`` means the
+        reported quota is exhausted — but an open grace window can still
+        admit uploads, since the console then enforces a temporary
+        ``grace_bytes`` ceiling instead of ``data_volume_limit``.
+        ``False`` does *not* guarantee one will succeed: the console also
+        hard-blocks on billing suspension and walks up parent allocations —
+        and the daemon sends none of those, so they are invisible here.
+        Treat a ``False`` as "the reported quota has room", not as a
+        guarantee.
         """
         return self.bytes_free == 0 or self.records_free == 0
 
@@ -296,9 +298,11 @@ class TransferSummary:
 
     Returned by :meth:`DataeraiClient.list_transfers`.  ``status`` is the
     daemon's own lifecycle state — ``queued``, ``active``, ``paused``,
-    ``completed`` or ``failed``; ``queued`` is real rather than notional,
-    because the daemon runs a bounded number of transfers at once and queues
-    the rest.
+    ``complete``, ``failed`` or ``cancelled``; ``queued`` is real rather than
+    notional, because the daemon runs a bounded number of transfers at once
+    and queues the rest.  ``complete``, ``failed`` and ``cancelled`` are
+    terminal; a ``cancelled`` transfer (see
+    :meth:`DataeraiClient.cancel_transfer`) is never resumed.
     """
 
     transfer_id: str

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/_version.py

@@ -9,4 +9,4 @@ See ``packaging/version.py`` (the ``compute_version`` brain) and
 ``doc/runbooks/20260616_SDK_CLI_PyPI_Release_Runbook.md``.
 """
 
-__version__ = "0.3.0b8"
+__version__ = "0.3.0.dev1"

{dataerai_sdk-0.3.0b8 → dataerai_sdk-0.3.0.dev1}/dataerai/bambu/fleet_hub.py

@@ -55,6 +55,17 @@ class HubTransport(Protocol):
     ) -> HubResponse: ...
 
 
+class _RejectRedirects(urllib.request.HTTPRedirectHandler):
+    """Keep bearer credentials and client-certificate use on the configured hub."""
+
+    def redirect_request(self, request, response, code, message, headers, new_url):
+        # urllib otherwise forwards Authorization even across origins and from
+        # HTTPS to HTTP. Refuse same-origin redirects too: an API endpoint must
+        # not silently change the method or replay a credential-bearing request.
+        response.close()
+        raise FleetHubError("Fleet Hub redirects are not allowed", status=code)
+
+
 class UrlLibTransport:
     """Small mTLS transport kept behind a protocol for deterministic tests."""
 
@@ -77,7 +88,9 @@ class UrlLibTransport:
         self.timeout_s = timeout_s
         context = ssl.create_default_context(cafile=ca_cert)
         context.load_cert_chain(certfile=client_cert, keyfile=client_key)
-        self._opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=context))
+        self._opener = urllib.request.build_opener(
+            urllib.request.HTTPSHandler(context=context), _RejectRedirects()
+        )
 
     def request(
         self,

dataerai_sdk-0.3.0.dev1/dataerai/chat_capture/__init__.py

@@ -0,0 +1 @@
+"""Opt-in conversation capture; provider SDKs are never monkey patched."""

dataerai_sdk-0.3.0.dev1/dataerai/chat_capture/anthropic.py

@@ -0,0 +1,56 @@
+"""Observe Claude Messages create, preserving indexed block order and SDK values."""
+import asyncio
+import json
+from .recorder import blocks, plain, safe, ObservedStream, AsyncObservedStream
+
+
+def outcome(reason):
+    return {'end_turn':'ok','stop_sequence':'ok','tool_use':'ok','max_tokens':'truncated',
+            'model_context_window_exceeded':'truncated','refusal':'refused'}.get(reason,'unknown')
+
+
+class ClaudeState:
+    def __init__(self,run):self.run=run;self.parts={};self.arguments={};self.reason=None
+    def complete(self,response):
+        value=plain(response);self.run.output(0,blocks(value.get('content')),outcome(value.get('stop_reason')))
+    def consume(self,event):
+        value=plain(event);kind=value.get('type')
+        if kind=='message_start':return
+        if kind=='error':self.run.finish('error');return
+        if kind=='message_delta':self.reason=value.get('delta',{}).get('stop_reason',self.reason);return
+        if kind=='message_stop':self.run.output(0,self.content(),outcome(self.reason));return
+        index=value.get('index')
+        if kind=='content_block_start':
+            content=blocks([value['content_block']])
+            if content:self.parts[index]=content[0]
+        elif kind=='content_block_delta':
+            delta=value['delta'];block=self.parts.get(index)
+            if not block:return  # hidden thinking/signature blocks have no entry
+            if delta['type']=='text_delta' and block['type']=='text':block['text']+=delta['text']
+            elif delta['type']=='input_json_delta' and block['type']=='tool_call':
+                self.arguments[index]=self.arguments.get(index,'')+delta['partial_json'];block['input']=self.arguments[index]
+            else:return
+        elif kind=='content_block_stop':
+            if index in self.arguments:
+                try:self.parts[index]['input']=json.loads(self.arguments[index])
+                except ValueError:self.parts[index]['input']={'raw_arguments':self.arguments[index]}
+        else:return  # ping and future metadata events do not become content
+        self.run.output(0,self.content())
+    def content(self):return [self.parts[i] for i in sorted(self.parts)]
+
+
+class ClaudeObserver:
+    def __init__(self,recorder):self.recorder=recorder
+    def messages(self,call,**kwargs):
+        run=self.recorder.start('anthropic',kwargs,'chat');state=ClaudeState(run)
+        try:result=call(**kwargs)
+        except BaseException:run.finish('error');raise
+        if kwargs.get('stream'):return ObservedStream(result,run,state.consume)
+        safe(run,state.complete,result);return result
+    async def amessages(self,call,**kwargs):
+        run=await asyncio.to_thread(self.recorder.start,'anthropic',kwargs,'chat');state=ClaudeState(run)
+        try:result=await call(**kwargs)
+        except asyncio.CancelledError:await asyncio.to_thread(run.finish,'cancelled');raise
+        except BaseException:await asyncio.to_thread(run.finish,'error');raise
+        if kwargs.get('stream'):return AsyncObservedStream(result,run,state.consume)
+        await asyncio.to_thread(safe,run,state.complete,result);return result