dataerai-sdk 0.2.0b20__tar.gz → 0.2.0.dev1__tar.gz

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataerai-sdk
-Version: 0.2.0b20
+Version: 0.2.0.dev1
 Summary: Python SDK for the Dataerai transfer daemon
 Author: Dataerai
 License: DATAERAI PROPRIETARY SOFTWARE LICENSE AGREEMENT
@@ -370,15 +370,6 @@ License-File: LICENSE
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
 Requires-Dist: pytest-mock>=3; extra == "dev"
-Requires-Dist: ipython>=8; extra == "dev"
-Provides-Extra: notebook
-Requires-Dist: ipython>=8; extra == "notebook"
-Provides-Extra: nn-tensorflow
-Requires-Dist: ipython>=8; extra == "nn-tensorflow"
-Requires-Dist: tensorflow<2.19,>=2.18; extra == "nn-tensorflow"
-Requires-Dist: numpy<3,>=2; extra == "nn-tensorflow"
-Requires-Dist: psutil>=5.9; extra == "nn-tensorflow"
-Requires-Dist: nvidia-ml-py>=12; extra == "nn-tensorflow"
 Provides-Extra: bambu
 Requires-Dist: imageio-ffmpeg>=0.6; extra == "bambu"
 Provides-Extra: ml-io
@@ -495,8 +486,6 @@ Optional ML adapters are installed separately so the base SDK stays small:
 pip install --pre "dataerai-sdk[ml]>=0.2.0b1,<0.3"          # PyTorch Zarr datasets
 pip install --pre "dataerai-sdk[ml-keras]>=0.2.0b1,<0.3"    # Keras batches
 pip install --pre "dataerai-sdk[ml-hls4ml]>=0.2.0b1,<0.3"   # HLS4ML conversion
-pip install --pre "dataerai-sdk[notebook]>=0.2.0b1,<0.3"    # IPython %dataerai magic
-pip install --pre "dataerai-sdk[nn-tensorflow]>=0.2.0b1,<0.3" # TensorFlow provenance
 ```
 
 ## Quick start
@@ -548,126 +537,6 @@ with DataeraiClient(binary_path="/usr/local/bin/dataerai") as client:
     print(f"Linked via {relationship.type} to {related_id}")
 ```
 
-## Notebook magic
-
-After signing in with the Dataerai CLI, a notebook only needs a destination
-path. The first component names the project; the notebook magic creates that
-project when it is missing, then creates any missing collection components
-beneath it:
-
-```python
-DESTINATION_COLLECTION_PATH = "Research / Experiments / July"
-
-%load_ext dataerai.magics
-%dataerai $DESTINATION_COLLECTION_PATH
-```
-
-The magic publishes a `dataerai_session` variable. Its uploads automatically
-target the selected project and collection:
-
-```python
-dataset = dataerai_session.find_asset(
-    "did:dataerai:beta:asset:...",
-    title="input.csv",
-)
-dataerai_session.download(dataset, "source-data")
-result = dataerai_session.upload(
-    "analysis.csv",
-    metadata={"component": "derived-output"},
-)
-```
-
-Use `%dataerai --as run $DESTINATION_COLLECTION_PATH` to choose a different
-session variable. Regular Python code can call
-`connect_notebook(DESTINATION_COLLECTION_PATH)` instead.
-
-### Trace a notebook run
-
-Add `--trace` to capture every subsequent cell's source, timestamps, stdout,
-stderr, structured Python logging records, rich display outputs, returned
-value, and error. Uploads and downloads through the session join the same run
-automatically, following the run-centered provenance pattern used by the QICK
-integration:
-
-```python
-%dataerai --trace --notebook analysis.ipynb --title "July analysis" $DESTINATION_COLLECTION_PATH
-
-# Run analysis cells and upload their products through dataerai_session.
-result = dataerai_session.upload(
-    "analysis.csv",
-    record_type="analysis",
-    metadata={"component": "derived-output"},
-)
-
-%dataerai --finish
-```
-
-`%dataerai --finish` waits until its cell completes, then uploads a JSON
-execution-log asset with `record_type="log"`. Every uploaded asset carries a
-shared `notebook_run_id` plus `dataerai-notebook-trace` and
-`notebook-run:<run-id>` tags. The log is linked to each recorded input and
-product with `records_telemetry`, and the published result is available as
-`dataerai_trace`. Run identity is merged onto an existing same-title asset, so
-re-running a fixed notebook or output filename keeps both run tags searchable.
-If a traced cell fails, its source, outputs, error, and traceback are published
-immediately; an explicit `%dataerai --finish` is not required for that failed
-run.
-
-Tracing is deliberately opt-in because cell source and output can contain
-secrets or sensitive research data. Review the notebook before enabling it.
-The recorder does not read environment-variable values or scan/upload arbitrary
-files; a file becomes a recorded product only when the notebook uploads it
-through the traced session. Finish an active trace before starting another one
-in the same Python kernel.
-
-## TensorFlow neural-network provenance
-
-`TensorFlowProvenanceTracker` implements the complete preservation contract
-ported from
-[DataFed_TorchFlow](https://github.com/m3-learning/DataFed_TorchFlow). It saves
-native `.keras` checkpoints containing model and optimizer state, writes a
-machine-readable manifest, captures architecture and hyperparameters, data and
-training-code checksums, outcomes, and full runtime/system properties, and
-authors `derived-from` links to datasets, the notebook, and prior checkpoints.
-
-```python
-from dataerai.nn import TensorFlowProvenanceTracker
-
-tracker = TensorFlowProvenanceTracker(
-    model,
-    session=dataerai_session,
-    run_name="mnist-mlp",
-    dataset_references=[
-        {
-            "asset_id": dataset_asset.asset_id,
-            "source_asset_did": SOURCE_ASSET_DID,
-            "filename": DATA_PATH.name,
-            "sha256": dataset_sha,
-        }
-    ],
-    notebook_path=NOTEBOOK_PATH,
-    notebook_asset_id=notebook_asset.asset_id,
-)
-history = model.fit(
-    x_train,
-    y_train,
-    epochs=2,
-    callbacks=[tracker.callback(training_parameters={"batch_size": 128})],
-)
-final = tracker.save_checkpoint(
-    epoch=2,
-    label="final",
-    record_name="mnist-mlp",
-    metrics={"test_accuracy": 0.91},
-    training_parameters={"batch_size": 128, "epochs": 2},
-)
-```
-
-System capture intentionally does not enumerate environment variables. It
-records OS/host, CPU and memory properties, TensorFlow-visible devices,
-optional NVML GPU memory, Python/framework/package versions, and only a small
-allowlist of determinism and thread settings.
-
 ## API reference
 
 ### `DataeraiClient(*, socket_path, binary_path, auto_start, start_timeout_s, request_timeout_s)`
@@ -687,11 +556,6 @@ Use as a context manager (`with DataeraiClient(...) as client:`) for automatic c
 | Method | Returns | Description |
 |---|---|---|
 | `auth_status()` | `AuthStatus` | Logged-in user ID, email, and token expiry; `user_id` is `None` with older daemons |
-| `list_tree()` | `list[TreeNode]` | List visible workspaces, projects, and collections |
-| `create_collection(title, *, owner_type, owner_id, parent_id=None)` | `Collection` | Create a collection |
-| `ensure_collection_path(path, *, create_project=False, project_description="")` | `CollectionDestination` | Resolve `Project / Collection / ...`, creating missing collection components and optionally the project |
-| `search_assets(query, ...)` | `AssetSearchPage` | Search one page of visible assets |
-| `find_assets(query, ...)` | `list[AssetSummary]` | Search and collect every result page |
 | `create_project(name, *, description="")` | `Project` | Create a project (+ root collection, owner membership, default allocation); requires write scope |
 | `upload(local_path, *, title, owner_type, owner_id, record_type=None, ...)` | `UploadResult` | Upload a file with an optional server-validated record type; blocks until complete |
 | `download(asset_id, dest_dir, ...)` | `DownloadResult` | Download latest asset content; blocks until complete |

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/README.md

@@ -24,8 +24,6 @@ Optional ML adapters are installed separately so the base SDK stays small:
 pip install --pre "dataerai-sdk[ml]>=0.2.0b1,<0.3"          # PyTorch Zarr datasets
 pip install --pre "dataerai-sdk[ml-keras]>=0.2.0b1,<0.3"    # Keras batches
 pip install --pre "dataerai-sdk[ml-hls4ml]>=0.2.0b1,<0.3"   # HLS4ML conversion
-pip install --pre "dataerai-sdk[notebook]>=0.2.0b1,<0.3"    # IPython %dataerai magic
-pip install --pre "dataerai-sdk[nn-tensorflow]>=0.2.0b1,<0.3" # TensorFlow provenance
 ```
 
 ## Quick start
@@ -77,126 +75,6 @@ with DataeraiClient(binary_path="/usr/local/bin/dataerai") as client:
     print(f"Linked via {relationship.type} to {related_id}")
 ```
 
-## Notebook magic
-
-After signing in with the Dataerai CLI, a notebook only needs a destination
-path. The first component names the project; the notebook magic creates that
-project when it is missing, then creates any missing collection components
-beneath it:
-
-```python
-DESTINATION_COLLECTION_PATH = "Research / Experiments / July"
-
-%load_ext dataerai.magics
-%dataerai $DESTINATION_COLLECTION_PATH
-```
-
-The magic publishes a `dataerai_session` variable. Its uploads automatically
-target the selected project and collection:
-
-```python
-dataset = dataerai_session.find_asset(
-    "did:dataerai:beta:asset:...",
-    title="input.csv",
-)
-dataerai_session.download(dataset, "source-data")
-result = dataerai_session.upload(
-    "analysis.csv",
-    metadata={"component": "derived-output"},
-)
-```
-
-Use `%dataerai --as run $DESTINATION_COLLECTION_PATH` to choose a different
-session variable. Regular Python code can call
-`connect_notebook(DESTINATION_COLLECTION_PATH)` instead.
-
-### Trace a notebook run
-
-Add `--trace` to capture every subsequent cell's source, timestamps, stdout,
-stderr, structured Python logging records, rich display outputs, returned
-value, and error. Uploads and downloads through the session join the same run
-automatically, following the run-centered provenance pattern used by the QICK
-integration:
-
-```python
-%dataerai --trace --notebook analysis.ipynb --title "July analysis" $DESTINATION_COLLECTION_PATH
-
-# Run analysis cells and upload their products through dataerai_session.
-result = dataerai_session.upload(
-    "analysis.csv",
-    record_type="analysis",
-    metadata={"component": "derived-output"},
-)
-
-%dataerai --finish
-```
-
-`%dataerai --finish` waits until its cell completes, then uploads a JSON
-execution-log asset with `record_type="log"`. Every uploaded asset carries a
-shared `notebook_run_id` plus `dataerai-notebook-trace` and
-`notebook-run:<run-id>` tags. The log is linked to each recorded input and
-product with `records_telemetry`, and the published result is available as
-`dataerai_trace`. Run identity is merged onto an existing same-title asset, so
-re-running a fixed notebook or output filename keeps both run tags searchable.
-If a traced cell fails, its source, outputs, error, and traceback are published
-immediately; an explicit `%dataerai --finish` is not required for that failed
-run.
-
-Tracing is deliberately opt-in because cell source and output can contain
-secrets or sensitive research data. Review the notebook before enabling it.
-The recorder does not read environment-variable values or scan/upload arbitrary
-files; a file becomes a recorded product only when the notebook uploads it
-through the traced session. Finish an active trace before starting another one
-in the same Python kernel.
-
-## TensorFlow neural-network provenance
-
-`TensorFlowProvenanceTracker` implements the complete preservation contract
-ported from
-[DataFed_TorchFlow](https://github.com/m3-learning/DataFed_TorchFlow). It saves
-native `.keras` checkpoints containing model and optimizer state, writes a
-machine-readable manifest, captures architecture and hyperparameters, data and
-training-code checksums, outcomes, and full runtime/system properties, and
-authors `derived-from` links to datasets, the notebook, and prior checkpoints.
-
-```python
-from dataerai.nn import TensorFlowProvenanceTracker
-
-tracker = TensorFlowProvenanceTracker(
-    model,
-    session=dataerai_session,
-    run_name="mnist-mlp",
-    dataset_references=[
-        {
-            "asset_id": dataset_asset.asset_id,
-            "source_asset_did": SOURCE_ASSET_DID,
-            "filename": DATA_PATH.name,
-            "sha256": dataset_sha,
-        }
-    ],
-    notebook_path=NOTEBOOK_PATH,
-    notebook_asset_id=notebook_asset.asset_id,
-)
-history = model.fit(
-    x_train,
-    y_train,
-    epochs=2,
-    callbacks=[tracker.callback(training_parameters={"batch_size": 128})],
-)
-final = tracker.save_checkpoint(
-    epoch=2,
-    label="final",
-    record_name="mnist-mlp",
-    metrics={"test_accuracy": 0.91},
-    training_parameters={"batch_size": 128, "epochs": 2},
-)
-```
-
-System capture intentionally does not enumerate environment variables. It
-records OS/host, CPU and memory properties, TensorFlow-visible devices,
-optional NVML GPU memory, Python/framework/package versions, and only a small
-allowlist of determinism and thread settings.
-
 ## API reference
 
 ### `DataeraiClient(*, socket_path, binary_path, auto_start, start_timeout_s, request_timeout_s)`
@@ -216,11 +94,6 @@ Use as a context manager (`with DataeraiClient(...) as client:`) for automatic c
 | Method | Returns | Description |
 |---|---|---|
 | `auth_status()` | `AuthStatus` | Logged-in user ID, email, and token expiry; `user_id` is `None` with older daemons |
-| `list_tree()` | `list[TreeNode]` | List visible workspaces, projects, and collections |
-| `create_collection(title, *, owner_type, owner_id, parent_id=None)` | `Collection` | Create a collection |
-| `ensure_collection_path(path, *, create_project=False, project_description="")` | `CollectionDestination` | Resolve `Project / Collection / ...`, creating missing collection components and optionally the project |
-| `search_assets(query, ...)` | `AssetSearchPage` | Search one page of visible assets |
-| `find_assets(query, ...)` | `list[AssetSummary]` | Search and collect every result page |
 | `create_project(name, *, description="")` | `Project` | Create a project (+ root collection, owner membership, default allocation); requires write scope |
 | `upload(local_path, *, title, owner_type, owner_id, record_type=None, ...)` | `UploadResult` | Upload a file with an optional server-validated record type; blocks until complete |
 | `download(asset_id, dest_dir, ...)` | `DownloadResult` | Download latest asset content; blocks until complete |

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/dataerai/__init__.py

@@ -3,25 +3,18 @@
 from ._version import __version__
 from .client import DataeraiClient
 from ._agent import DataeraiAgent, run_agent
-from ._errors import DaemonError, DaemonTimeoutError, TermsAcceptanceRequired
+from ._errors import DaemonError, DaemonTimeoutError
 from ._types import (
     AssetMetadata,
-    AssetSearchPage,
-    AssetSummary,
     AuthStatus,
-    Collection,
-    CollectionDestination,
     CollectionDownloadResult,
     DownloadedFile,
     DownloadResult,
     Project,
     ProgressEvent,
     Relationship,
-    TreeNode,
     UploadResult,
 )
-from .notebook import NotebookSession, connect_notebook
-from .notebook_trace import NotebookExecutionTrace, NotebookTraceResult
 
 __all__ = [
     "__version__",
@@ -30,14 +23,8 @@ __all__ = [
     "run_agent",
     "DaemonError",
     "DaemonTimeoutError",
-    "TermsAcceptanceRequired",
     "AuthStatus",
     "Project",
-    "Collection",
-    "TreeNode",
-    "CollectionDestination",
-    "AssetSummary",
-    "AssetSearchPage",
     "Relationship",
     "UploadResult",
     "DownloadedFile",
@@ -45,8 +32,4 @@ __all__ = [
     "CollectionDownloadResult",
     "ProgressEvent",
     "AssetMetadata",
-    "NotebookSession",
-    "NotebookExecutionTrace",
-    "NotebookTraceResult",
-    "connect_notebook",
 ]

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/dataerai/_daemon.py

@@ -3,39 +3,23 @@
 from __future__ import annotations
 
 import os
-import re
 import subprocess
-import sys
 import tempfile
 import time
 from pathlib import Path
 
-from ._transport import endpoint_exists
-
 
 def default_socket_path() -> str:
-    """Return the default daemon IPC endpoint for this platform.
+    """Return the default Unix socket path.
 
     Resolution order:
     1. ``$DATAERAI_SOCKET`` environment variable
-    2. Windows: ``\\\\.\\pipe\\dataerai-transfer-<USERNAME>`` (a named pipe —
-       the daemon has no usable AF_UNIX transport there)
-    3. ``$XDG_RUNTIME_DIR/dataerai-transfer.sock``  (Linux / systemd)
-    4. ``<tempdir>/dataerai-transfer.sock``          (macOS and other platforms)
-
-    The Windows derivation is kept in lockstep with the daemon's own default
-    (``cli/cmd/dataerai-transfer/main.go`` ``defaultSocketPath``) and the Node
-    SDK (``sdk/nodejs/src/daemon.ts``): same ``USERNAME`` source, same
-    ``[^A-Za-z0-9._-] -> _`` sanitisation, so all three resolve to one pipe.
+    2. ``$XDG_RUNTIME_DIR/dataerai-transfer.sock``  (Linux / systemd)
+    3. ``<tempdir>/dataerai-transfer.sock``          (macOS and other platforms)
     """
     env = os.environ.get("DATAERAI_SOCKET")
     if env:
         return env
-    if sys.platform == "win32":
-        # Per-user suffix so concurrent Windows sessions do not collide in the
-        # machine-global pipe namespace.
-        user = re.sub(r"[^A-Za-z0-9._-]", "_", os.environ.get("USERNAME") or "default")
-        return rf"\\.\pipe\dataerai-transfer-{user}"
     xdg = os.environ.get("XDG_RUNTIME_DIR")
     if xdg:
         return str(Path(xdg) / "dataerai-transfer.sock")
@@ -59,15 +43,12 @@ def spawn_daemon(binary_path: str, socket_path: str) -> subprocess.Popen:
 
 
 def wait_for_socket(socket_path: str, timeout_s: float = 10.0) -> None:
-    """Poll until the daemon endpoint at *socket_path* is available.
+    """Poll until *socket_path* exists or *timeout_s* elapses.
 
-    Raises :class:`TimeoutError` if it does not appear in time. On POSIX
-    "available" means the AF_UNIX socket file exists; on Windows a named pipe is
-    not in the filesystem namespace, so ``endpoint_exists`` probes for a live
-    listener instead (see :func:`dataerai._transport.endpoint_exists`).
+    Raises :class:`TimeoutError` if the socket does not appear in time.
     """
     deadline = time.monotonic() + timeout_s
-    while not endpoint_exists(socket_path):
+    while not Path(socket_path).exists():
         if time.monotonic() >= deadline:
             raise TimeoutError(
                 f"Daemon socket '{socket_path}' not available after {timeout_s:.0f}s. "

dataerai_sdk-0.2.0.dev1/dataerai/_errors.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+
+class DaemonError(Exception):
+    """Raised when the daemon returns a coded error response."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return f"DaemonError(code={self.code!r}, message={self.message!r})"
+
+
+class DaemonTimeoutError(TimeoutError):
+    """Raised when a request to the daemon (or a transfer) exceeds its timeout."""

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/dataerai/_types.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any, Optional
+from typing import Any, Callable, Optional
 
 
 # ─── Auth ─────────────────────────────────────────────────────────────────────
@@ -39,67 +39,6 @@ class Project:
     created_at: Optional[str] = None
 
 
-# ─── Project tree and collections ─────────────────────────────────────────────
-
-
-@dataclass
-class Collection:
-    """A collection returned by the project-tree and create APIs."""
-
-    collection_id: str
-    title: str
-    parent_id: Optional[str] = None
-    owner_user_id: Optional[str] = None
-    owner_project_id: Optional[str] = None
-    is_root: bool = False
-
-
-@dataclass
-class TreeNode:
-    """A user workspace or project in :meth:`DataeraiClient.list_tree`."""
-
-    type: str
-    id: str
-    name: str
-    root_collection_id: Optional[str] = None
-    collections: list[Collection] = field(default_factory=list)
-
-
-@dataclass
-class CollectionDestination:
-    """A name-resolved project collection created by ``ensure_collection_path``."""
-
-    path: str
-    project_id: str
-    project_name: str
-    collection_id: str
-
-
-# ─── Asset search ──────────────────────────────────────────────────────────────
-
-
-@dataclass
-class AssetSummary:
-    """One asset returned by collection listing or global search."""
-
-    asset_id: str
-    title: str
-    alias: Optional[str] = None
-    record_type: Optional[str] = None
-    size_bytes: int = 0
-    has_content: bool = False
-    updated_at: Optional[str] = None
-
-
-@dataclass
-class AssetSearchPage:
-    """One page returned by :meth:`DataeraiClient.search_assets`."""
-
-    assets: list[AssetSummary] = field(default_factory=list)
-    has_more: bool = False
-    next_cursor: Optional[str] = None
-
-
 # ─── Provenance relationship ────────────────────────────────────────────────────
 
 

{dataerai_sdk-0.2.0b20 → dataerai_sdk-0.2.0.dev1}/dataerai/_version.py

@@ -9,4 +9,4 @@ See ``packaging/version.py`` (the ``compute_version`` brain) and
 ``doc/runbooks/20260616_SDK_CLI_PyPI_Release_Runbook.md``.
 """
 
-__version__ = "0.2.0b20"
+__version__ = "0.2.0.dev1"