dataerai-sdk 0.2.0.dev1__tar.gz → 0.3.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataerai_sdk-0.3.0b1/PKG-INFO +1630 -0
- dataerai_sdk-0.3.0b1/README.md +1131 -0
- dataerai_sdk-0.3.0b1/dataerai/__init__.py +128 -0
- dataerai_sdk-0.3.0b1/dataerai/_daemon.py +67 -0
- dataerai_sdk-0.3.0b1/dataerai/_darwin_acl_probe.py +56 -0
- dataerai_sdk-0.3.0b1/dataerai/_endpoint.py +237 -0
- dataerai_sdk-0.3.0b1/dataerai/_errors.py +78 -0
- dataerai_sdk-0.3.0b1/dataerai/_events.py +318 -0
- dataerai_sdk-0.3.0b1/dataerai/_hello.py +66 -0
- dataerai_sdk-0.3.0b1/dataerai/_hello_connection.py +34 -0
- dataerai_sdk-0.3.0b1/dataerai/_hello_exchange.py +167 -0
- dataerai_sdk-0.3.0b1/dataerai/_hello_key.py +152 -0
- dataerai_sdk-0.3.0b1/dataerai/_hello_key_location.py +81 -0
- dataerai_sdk-0.3.0b1/dataerai/_transport.py +318 -0
- dataerai_sdk-0.3.0b1/dataerai/_types.py +731 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/_version.py +1 -1
- dataerai_sdk-0.3.0b1/dataerai/_windows_identity.py +149 -0
- dataerai_sdk-0.3.0b1/dataerai/_windows_key.py +407 -0
- dataerai_sdk-0.3.0b1/dataerai/_windows_key_acl.py +76 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/__init__.py +3 -0
- dataerai_sdk-0.3.0b1/dataerai/bambu/cli.py +171 -0
- dataerai_sdk-0.3.0b1/dataerai/bambu/feature_access.py +44 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/fleet_hub.py +22 -5
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/integration.py +27 -2
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/recorder.py +5 -0
- dataerai_sdk-0.3.0b1/dataerai/client.py +2854 -0
- dataerai_sdk-0.3.0b1/dataerai/envelope.py +843 -0
- dataerai_sdk-0.3.0b1/dataerai/magics.py +134 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/__init__.py +32 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/api.py +131 -9
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bases.py +98 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/containers/_netcdf_reader.py +130 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/netcdf.py +9 -6
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/core.py +45 -6
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/data_type_definitions.py +158 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/documents/__init__.py +97 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/documents/budget.py +359 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/documents/execution_log.py +131 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/documents/json_document.py +79 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/documents/notebook.py +321 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/feature_access.py +48 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/bruker/nanoscope.py +25 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/instruments/AFM/oxfordAFM/ibw.py +79 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/instruments/MS/__init__.py +1 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/instruments/MS/hiden/__init__.py +1 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/instruments/MS/hiden/massoft_csv.py +302 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/massspec/andi_ms.py +392 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/registry.py +235 -8
- dataerai_sdk-0.3.0b1/dataerai/metaextract/spectroscopy/jcamp.py +445 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/text_metadata.py +1121 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/metadata.py +218 -57
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/__init__.py +187 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/_common.py +147 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/metadata_schemas/afm.py +75 -2
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/bio.py +165 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/chemistry.py +90 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/clinical.py +111 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/containers.py +132 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/generic.py +131 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/imaging.py +196 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/microscopy.py +123 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/spectroscopy.py +70 -0
- dataerai_sdk-0.3.0b1/dataerai/metaextract/viz/metadata_schemas/xrd.py +106 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/access.py +160 -20
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/client.py +2 -1
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/croissant.py +1 -1
- dataerai_sdk-0.3.0b1/dataerai/ml/lineage.py +418 -0
- dataerai_sdk-0.3.0b1/dataerai/ml/serve/__init__.py +26 -0
- dataerai_sdk-0.3.0b1/dataerai/ml/serve/__main__.py +49 -0
- dataerai_sdk-0.3.0b1/dataerai/ml/serve/inputs.py +99 -0
- dataerai_sdk-0.3.0b1/dataerai/ml/serve/loader.py +309 -0
- dataerai_sdk-0.3.0b1/dataerai/ml/serve/runtime.py +412 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/staging.py +2 -2
- dataerai_sdk-0.3.0b1/dataerai/nn/__init__.py +22 -0
- dataerai_sdk-0.3.0b1/dataerai/nn/_provenance.py +191 -0
- dataerai_sdk-0.3.0b1/dataerai/nn/pytorch.py +631 -0
- dataerai_sdk-0.3.0b1/dataerai/nn/system.py +269 -0
- dataerai_sdk-0.3.0b1/dataerai/nn/tensorflow.py +567 -0
- dataerai_sdk-0.3.0b1/dataerai/notebook.py +284 -0
- dataerai_sdk-0.3.0b1/dataerai/notebook_trace.py +965 -0
- dataerai_sdk-0.3.0b1/dataerai/pycroscopy/__init__.py +13 -0
- dataerai_sdk-0.3.0b1/dataerai/pycroscopy/feature_control.py +120 -0
- dataerai_sdk-0.3.0b1/dataerai/qick/__init__.py +19 -0
- dataerai_sdk-0.3.0b1/dataerai/qick/feature_control.py +144 -0
- dataerai_sdk-0.3.0b1/dataerai/rest/__init__.py +40 -0
- dataerai_sdk-0.3.0b1/dataerai/rest/_config.py +95 -0
- dataerai_sdk-0.3.0b1/dataerai/rest/auth.py +433 -0
- dataerai_sdk-0.3.0b1/dataerai/rest/client.py +441 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/__init__.py +37 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/_winjob.py +178 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/agent.py +1023 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/api.py +65 -10
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/artifacts.py +59 -15
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/capabilities.py +123 -4
- dataerai_sdk-0.3.0b1/dataerai/runner/cli.py +414 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/config.py +86 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/cpu_profile.py +78 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/csv_extract.py +137 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/device_auth.py +208 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/downloads.py +258 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/executor.py +540 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/handlers/__init__.py +25 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/handlers/_harness.py +106 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/handlers/extract_csv.py +31 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/handlers/infer.py +19 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/handlers/script.py +596 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/handlers/serve.py +29 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/heartbeat.py +8 -2
- dataerai_sdk-0.3.0b1/dataerai/runner/image_ref.py +76 -0
- dataerai_sdk-0.3.0b1/dataerai/runner/sigv4.py +182 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/state.py +43 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/telemetry.py +44 -4
- dataerai_sdk-0.3.0b1/dataerai/runner/venvs.py +176 -0
- dataerai_sdk-0.3.0b1/dataerai/watermark.py +741 -0
- dataerai_sdk-0.3.0b1/dataerai_sdk.egg-info/PKG-INFO +1630 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai_sdk.egg-info/SOURCES.txt +110 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai_sdk.egg-info/requires.txt +45 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/pyproject.toml +66 -4
- dataerai_sdk-0.3.0b1/tests/test_attribution_feature_control.py +44 -0
- dataerai_sdk-0.3.0b1/tests/test_client.py +3408 -0
- dataerai_sdk-0.3.0b1/tests/test_collection_sync.py +498 -0
- dataerai_sdk-0.3.0b1/tests/test_collections.py +195 -0
- dataerai_sdk-0.3.0b1/tests/test_credentials_save_windows.py +174 -0
- dataerai_sdk-0.3.0b1/tests/test_daemon_startup_security.py +77 -0
- dataerai_sdk-0.3.0b1/tests/test_envelope.py +234 -0
- dataerai_sdk-0.3.0b1/tests/test_events.py +783 -0
- dataerai_sdk-0.3.0b1/tests/test_hello_client_security.py +245 -0
- dataerai_sdk-0.3.0b1/tests/test_hello_exchange_security.py +330 -0
- dataerai_sdk-0.3.0b1/tests/test_hello_key_location_security.py +66 -0
- dataerai_sdk-0.3.0b1/tests/test_hello_key_security.py +218 -0
- dataerai_sdk-0.3.0b1/tests/test_hello_proof_security.py +101 -0
- dataerai_sdk-0.3.0b1/tests/test_infer_handler.py +101 -0
- dataerai_sdk-0.3.0b1/tests/test_ipc_endpoint_security.py +252 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_lineage_client.py +4 -3
- dataerai_sdk-0.3.0b1/tests/test_lineage_duplicate_policy.py +442 -0
- dataerai_sdk-0.3.0b1/tests/test_nn_pytorch.py +759 -0
- dataerai_sdk-0.3.0b1/tests/test_nn_tensorflow.py +323 -0
- dataerai_sdk-0.3.0b1/tests/test_notebook.py +248 -0
- dataerai_sdk-0.3.0b1/tests/test_notebook_trace.py +615 -0
- dataerai_sdk-0.3.0b1/tests/test_notebook_trace_feature_control.py +376 -0
- dataerai_sdk-0.3.0b1/tests/test_provenance_tokens_feature_control.py +32 -0
- dataerai_sdk-0.3.0b1/tests/test_rest_auth.py +391 -0
- dataerai_sdk-0.3.0b1/tests/test_rest_client.py +361 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_inputs.py +57 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_loader.py +243 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_loader_device.py +161 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_packaging.py +65 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_runtime.py +267 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_runtime_hardening.py +254 -0
- dataerai_sdk-0.3.0b1/tests/test_serve_session.py +151 -0
- dataerai_sdk-0.3.0b1/tests/test_terms_gate.py +218 -0
- dataerai_sdk-0.3.0b1/tests/test_transport.py +378 -0
- dataerai_sdk-0.3.0b1/tests/test_tutorial_notebook.py +734 -0
- dataerai_sdk-0.3.0b1/tests/test_verifiable_credentials.py +97 -0
- dataerai_sdk-0.3.0b1/tests/test_watermark.py +536 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_key_acl_security.py +70 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_key_native_security.py +335 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_key_refusal_boundary.py +54 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_key_storage_security.py +234 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_pipe_cancellation_security.py +356 -0
- dataerai_sdk-0.3.0b1/tests/test_windows_pipe_identity_security.py +363 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_zarr_dataset.py +17 -6
- dataerai_sdk-0.2.0.dev1/PKG-INFO +0 -676
- dataerai_sdk-0.2.0.dev1/README.md +0 -214
- dataerai_sdk-0.2.0.dev1/dataerai/__init__.py +0 -35
- dataerai_sdk-0.2.0.dev1/dataerai/_daemon.py +0 -57
- dataerai_sdk-0.2.0.dev1/dataerai/_errors.py +0 -17
- dataerai_sdk-0.2.0.dev1/dataerai/_types.py +0 -165
- dataerai_sdk-0.2.0.dev1/dataerai/bambu/cli.py +0 -93
- dataerai_sdk-0.2.0.dev1/dataerai/client.py +0 -866
- dataerai_sdk-0.2.0.dev1/dataerai/metaextract/instruments/AFM/oxfordAFM/ibw.py +0 -55
- dataerai_sdk-0.2.0.dev1/dataerai/metaextract/spectroscopy/jcamp.py +0 -267
- dataerai_sdk-0.2.0.dev1/dataerai/metaextract/viz/metadata_schemas/__init__.py +0 -12
- dataerai_sdk-0.2.0.dev1/dataerai/ml/lineage.py +0 -244
- dataerai_sdk-0.2.0.dev1/dataerai/runner/__init__.py +0 -18
- dataerai_sdk-0.2.0.dev1/dataerai/runner/agent.py +0 -320
- dataerai_sdk-0.2.0.dev1/dataerai/runner/cli.py +0 -225
- dataerai_sdk-0.2.0.dev1/dataerai/runner/config.py +0 -41
- dataerai_sdk-0.2.0.dev1/dataerai/runner/device_auth.py +0 -119
- dataerai_sdk-0.2.0.dev1/dataerai/runner/executor.py +0 -127
- dataerai_sdk-0.2.0.dev1/dataerai/runner/sigv4.py +0 -96
- dataerai_sdk-0.2.0.dev1/dataerai_sdk.egg-info/PKG-INFO +0 -676
- dataerai_sdk-0.2.0.dev1/tests/test_client.py +0 -1188
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/LICENSE +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/_agent.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/_framing.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/_timeparse.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/capture.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/journal.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/bambu/models.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/alignment.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/fcs.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/gff.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/sequence.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/bio/vcf.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/checksum.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/compchem/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/compchem/molfile.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/compchem/qchem_output.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/compchem/structure.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/compchem/xyz.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/fits.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/hdf5.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/image.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/matlab.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/nexus.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/numpy_array.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/parquet.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/tiff.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/containers/zarr_store.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converter.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/AFM/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/AFM/oxfordAFM/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/AFM/oxfordAFM/ibw.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/RHEED/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/RHEED/ksa/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/RHEED/ksa/imm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/Xray/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/Xray/panalytical/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/Xray/panalytical/xrdml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/converters/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/crystallography/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/crystallography/cif.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/crystallography/vasp.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/data_type.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/bandexcitation/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/bandexcitation/h5.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/bruker/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/AFM/oxfordAFM/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/EM/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/EM/dm/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/EM/dm/dm3.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/EM/dm/dm4.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/RHEED/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/RHEED/ksa/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/RHEED/ksa/imm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/Xray/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/Xray/panalytical/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/Xray/panalytical/xrdml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/instruments/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/massspec/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/massspec/imzml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/massspec/mzml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/massspec/mzxml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/massspec/thermo_raw.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/medical/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/medical/dicom.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/medical/edf.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/medical/nifti.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/medical/nwb.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/microscopy/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/microscopy/czi.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/microscopy/leica_lif.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/microscopy/nd2.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/microscopy/olympus.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/spectroscopy/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/spectroscopy/bruker_opus.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/spectroscopy/renishaw_wdf.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/spectroscopy/thermo_spc.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/units.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/utils.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/Xray/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/Xray/panalytical/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/Xray/panalytical/xrd.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/afm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/dart_pfm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/single_frequency_pfm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/vector_pfm.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/instruments/oxford/afms/visualize.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/layout.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/maps.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/metaextract/viz/router.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/ai_ready_reader.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/calibration.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/codes.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/hooks.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/inference.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/sae.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/attribution/training.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/bench.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/cache.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/gpu.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/hls4ml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/io.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/keras_dataset.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/loader.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/report.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/__init__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/_docs.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/errors.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/live.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/reader.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/session.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/timeseries.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/video.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/stream/writer.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/telemetry.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/train_reference.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/ml/zarr_dataset.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/__main__.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/handlers/bench.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/handlers/stage.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai/runner/handlers/train.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai_sdk.egg-info/dependency_links.txt +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai_sdk.egg-info/entry_points.txt +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/dataerai_sdk.egg-info/top_level.txt +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/setup.cfg +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_agent.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ai_ready_croissant_external.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ai_ready_croissant_external_mlc.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_attribution_inference.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_attribution_sae.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_attribution_training.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_framing.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_keras_hls4ml.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_access_ttl.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_ai_ready_croissant.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_ai_ready_reader.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_bench.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_cache.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_content_pinning.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_cred_lifecycle.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_distributed.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_gpu.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_gpu_path.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_io.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_loader.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_packaging.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_read_path.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_review_followups.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_review_followups_torch.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_sampler_correctness.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_staging.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_ml_telemetry.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_stream_core.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_stream_live.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_stream_timeseries.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_stream_video.py +0 -0
- {dataerai_sdk-0.2.0.dev1 → dataerai_sdk-0.3.0b1}/tests/test_timeparse.py +0 -0
|
@@ -0,0 +1,1630 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dataerai-sdk
|
|
3
|
+
Version: 0.3.0b1
|
|
4
|
+
Summary: Python SDK for the Dataerai transfer daemon
|
|
5
|
+
Author: Dataerai
|
|
6
|
+
License: DATAERAI PROPRIETARY SOFTWARE LICENSE AGREEMENT
|
|
7
|
+
|
|
8
|
+
Version 1.2
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Dataerai, Inc. All Rights Reserved.
|
|
11
|
+
|
|
12
|
+
================================================================================
|
|
13
|
+
NOTICE: THE SOFTWARE IS PROPRIETARY AND CONFIDENTIAL. NO RIGHTS ARE GRANTED
|
|
14
|
+
EXCEPT AS EXPRESSLY AND NARROWLY SET FORTH BELOW. ALL RIGHTS NOT EXPRESSLY
|
|
15
|
+
GRANTED ARE RESERVED BY DATAERAI, INC. THIS AGREEMENT APPLIES EQUALLY TO USE
|
|
16
|
+
BY HUMANS AND BY ARTIFICIAL INTELLIGENCE AGENTS (SEE SECTION 4). IF YOU DO NOT
|
|
17
|
+
AGREE TO EVERY TERM OF THIS AGREEMENT, YOU HAVE NO LICENSE AND MUST NOT ACCESS,
|
|
18
|
+
INSTALL, COPY, OR USE THE SOFTWARE IN ANY MANNER.
|
|
19
|
+
================================================================================
|
|
20
|
+
|
|
21
|
+
This Dataerai Proprietary Software License Agreement (this "Agreement") is a
|
|
22
|
+
binding legal agreement between Dataerai, Inc., a corporation organized under
|
|
23
|
+
the laws of the Commonwealth of Pennsylvania ("Licensor"), and the single
|
|
24
|
+
identified individual or legal entity that has been expressly authorized in
|
|
25
|
+
writing by Licensor to receive the Software ("Licensee"). By accessing,
|
|
26
|
+
installing, copying, or using the Software, or by clicking to accept, Licensee
|
|
27
|
+
agrees to be bound by this Agreement.
|
|
28
|
+
|
|
29
|
+
--------------------------------------------------------------------------------
|
|
30
|
+
1. DEFINITIONS
|
|
31
|
+
--------------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
1.1 "Software" means the Dataerai software package, in any and all forms,
|
|
34
|
+
including without limitation source code, object code, byte code, binaries,
|
|
35
|
+
scripts, schemas, configuration, models, weights, prompts, embeddings,
|
|
36
|
+
graph taxonomies, registries, metadata, libraries, application programming
|
|
37
|
+
interfaces, command-line tooling, and any associated materials, together
|
|
38
|
+
with all updates, upgrades, patches, modifications, enhancements, and
|
|
39
|
+
Documentation, whether delivered now or in the future.
|
|
40
|
+
|
|
41
|
+
1.2 "Documentation" means any technical or user materials, in any medium,
|
|
42
|
+
provided or made available by Licensor relating to the Software.
|
|
43
|
+
|
|
44
|
+
1.3 "Authorized Purpose" means solely the internal, non-production evaluation
|
|
45
|
+
of the Software by Licensee, on systems owned and controlled by Licensee,
|
|
46
|
+
strictly within the scope, seat count, term, and field of use expressly
|
|
47
|
+
stated in a written authorization signed by an officer of Licensor. Absent
|
|
48
|
+
such a written authorization, the Authorized Purpose is null and no use is
|
|
49
|
+
permitted.
|
|
50
|
+
|
|
51
|
+
1.4 "Confidential Information" means the Software and any non-public information
|
|
52
|
+
disclosed by Licensor, whether or not marked confidential. The Software is
|
|
53
|
+
deemed Confidential Information in its entirety.
|
|
54
|
+
|
|
55
|
+
--------------------------------------------------------------------------------
|
|
56
|
+
2. GRANT OF LICENSE
|
|
57
|
+
--------------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
2.1 Subject to Licensee's strict and continuous compliance with every term of
|
|
60
|
+
this Agreement, Licensor grants to Licensee a personal, non-exclusive,
|
|
61
|
+
non-transferable, non-sublicensable, non-assignable, revocable, royalty-
|
|
62
|
+
bearing-or-evaluation-only, and fully revocable license to use the Software
|
|
63
|
+
solely for the Authorized Purpose and solely during the Term.
|
|
64
|
+
|
|
65
|
+
2.2 The license granted in Section 2.1 is the entirety of the rights granted.
|
|
66
|
+
It conveys no right to copy (except a single archival copy as required for
|
|
67
|
+
backup, retaining all notices), no right to modify, no right to create
|
|
68
|
+
derivative works, no right to distribute, no right to sublicense, no right
|
|
69
|
+
to display or perform publicly, and no right to use in production, in a
|
|
70
|
+
service bureau, or on behalf of any third party.
|
|
71
|
+
|
|
72
|
+
2.3 No license, immunity, or other right is granted by implication, estoppel,
|
|
73
|
+
exhaustion, or otherwise. No patent, trademark, or other intellectual
|
|
74
|
+
property right is licensed except the limited use right in Section 2.1.
|
|
75
|
+
|
|
76
|
+
--------------------------------------------------------------------------------
|
|
77
|
+
3. RESTRICTIONS
|
|
78
|
+
--------------------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
Licensee shall not, and shall not permit, enable, or assist any third party or
|
|
81
|
+
AI Agent (as defined in Section 4) to:
|
|
82
|
+
|
|
83
|
+
3.1 copy, reproduce, republish, upload, post, transmit, or otherwise duplicate
|
|
84
|
+
the Software except for the single archival copy permitted in Section 2.2;
|
|
85
|
+
|
|
86
|
+
3.2 modify, adapt, translate, port, fork, or create derivative works of the
|
|
87
|
+
Software, in whole or in part;
|
|
88
|
+
|
|
89
|
+
3.3 sell, resell, rent, lease, lend, distribute, transfer, disclose, host,
|
|
90
|
+
sublicense, time-share, or otherwise make the Software available to any
|
|
91
|
+
third party, whether for value or not;
|
|
92
|
+
|
|
93
|
+
3.4 reverse engineer, decompile, disassemble, decrypt, extract, or otherwise
|
|
94
|
+
attempt to derive or reconstruct the source code, architecture, algorithms,
|
|
95
|
+
model weights, training data, graph taxonomies, or trade secrets embodied
|
|
96
|
+
in the Software, except, and only to the minimum extent, that this
|
|
97
|
+
prohibition is unenforceable under applicable law and only after written
|
|
98
|
+
notice to Licensor;
|
|
99
|
+
|
|
100
|
+
3.5 use the Software to develop, train, benchmark, or improve any competing,
|
|
101
|
+
similar, or alternative product, model, or service;
|
|
102
|
+
|
|
103
|
+
3.6 conduct, publish, or disclose any benchmark, performance, security, or
|
|
104
|
+
comparative test or analysis of the Software without Licensor's prior
|
|
105
|
+
written consent;
|
|
106
|
+
|
|
107
|
+
3.7 remove, alter, obscure, or fail to reproduce any copyright, proprietary,
|
|
108
|
+
confidentiality, or other notice contained in or on the Software;
|
|
109
|
+
|
|
110
|
+
3.8 circumvent, disable, or interfere with any license-key, telemetry, usage-
|
|
111
|
+
metering, digital-rights-management, or access-control mechanism;
|
|
112
|
+
|
|
113
|
+
3.9 use the Software in excess of the authorized seat count, scope, field of
|
|
114
|
+
use, or Term, or for any purpose other than the Authorized Purpose; or
|
|
115
|
+
|
|
116
|
+
3.10 use the Software in violation of any applicable law, regulation, or third-
|
|
117
|
+
party right, including export control and sanctions laws.
|
|
118
|
+
|
|
119
|
+
--------------------------------------------------------------------------------
|
|
120
|
+
4. ARTIFICIAL INTELLIGENCE, AGENTS, AND AUTOMATED USE
|
|
121
|
+
--------------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
4.1 Definition. "AI Agent" means any artificial intelligence or machine-
|
|
124
|
+
learning system, model, or software — including any autonomous or semi-
|
|
125
|
+
autonomous agent, large language model, multi-agent system, robotic process
|
|
126
|
+
automation, bot, crawler, scraper, copilot, or other automated process —
|
|
127
|
+
that accesses, operates, invokes, queries, or acts upon the Software,
|
|
128
|
+
whether or not under direct human supervision, and whether operating on
|
|
129
|
+
Licensee's behalf, on Licensee's systems, or using Licensee's credentials,
|
|
130
|
+
tokens, API keys, or sessions.
|
|
131
|
+
|
|
132
|
+
4.2 Agents Bound by this Agreement. This Agreement applies in full to any access
|
|
133
|
+
to or use of the Software by, through, or on behalf of an AI Agent. Every
|
|
134
|
+
restriction, obligation, and prohibition that binds Licensee binds equally
|
|
135
|
+
any AI Agent that Licensee deploys, authorizes, instructs, integrates, or
|
|
136
|
+
permits to interact with the Software. An AI Agent is not a separate,
|
|
137
|
+
independent, or exempt user, and acquires no rights of its own under this
|
|
138
|
+
Agreement.
|
|
139
|
+
|
|
140
|
+
4.3 Full Responsibility for Agent Conduct. Licensee is fully and solely
|
|
141
|
+
responsible for all acts and omissions of any AI Agent acting on Licensee's
|
|
142
|
+
behalf, on Licensee's systems, or via Licensee's credentials, as if those
|
|
143
|
+
acts were Licensee's own. The autonomy of the AI Agent, the absence of human
|
|
144
|
+
supervision, or the absence of specific human intent is not a defense to any
|
|
145
|
+
breach. Licensee shall not use an AI Agent to do, attempt, or enable
|
|
146
|
+
anything that Licensee is itself prohibited from doing under this Agreement.
|
|
147
|
+
|
|
148
|
+
4.4 No Training, Ingestion, or Derivation. Licensee shall not, and shall not
|
|
149
|
+
permit any AI Agent or third party to, use the Software or any portion of it
|
|
150
|
+
— including its source code, object code, structure, schemas, graph
|
|
151
|
+
taxonomies, registries, metadata, model weights, prompts, embeddings,
|
|
152
|
+
Documentation, or Outputs — as input to, as training, fine-tuning, or
|
|
153
|
+
alignment data for, as a retrieval or context corpus for, or as a basis to
|
|
154
|
+
distill, replicate, or derive, any AI Agent, model, dataset, or competing
|
|
155
|
+
system.
|
|
156
|
+
|
|
157
|
+
4.5 No Automated Extraction. Licensee shall not use any AI Agent to scrape,
|
|
158
|
+
crawl, harvest, index, reverse engineer, or otherwise extract the Software,
|
|
159
|
+
its structure, its parameters, or its Outputs, including by means of
|
|
160
|
+
automated or adversarial querying intended to reconstruct or approximate the
|
|
161
|
+
Software's behavior, weights, taxonomies, or underlying data.
|
|
162
|
+
|
|
163
|
+
4.6 Outputs. "Outputs" means any data, content, predictions, classifications,
|
|
164
|
+
embeddings, analyses, graphs, or other results generated by the Software or
|
|
165
|
+
by any AI Agent through use of the Software. Outputs constitute Confidential
|
|
166
|
+
Information, are licensed (not assigned) to Licensee solely for the
|
|
167
|
+
Authorized Purpose, and remain subject to every restriction in this
|
|
168
|
+
Agreement. Licensee obtains no right to use Outputs to train, evaluate, or
|
|
169
|
+
improve any AI Agent, model, or competing system.
|
|
170
|
+
|
|
171
|
+
4.7 Disclosure of Agentic Use. Upon Licensor's request, Licensee shall disclose
|
|
172
|
+
whether, and the manner in which, any AI Agent has accessed or used the
|
|
173
|
+
Software, and shall cooperate with any audit under Section 8 directed at
|
|
174
|
+
such use.
|
|
175
|
+
|
|
176
|
+
--------------------------------------------------------------------------------
|
|
177
|
+
5. OWNERSHIP
|
|
178
|
+
--------------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
5.1 The Software is licensed, not sold. Licensor and its licensors retain all
|
|
181
|
+
right, title, and interest in and to the Software and all intellectual
|
|
182
|
+
property rights therein. Licensee acquires no ownership interest of any
|
|
183
|
+
kind. All feedback, suggestions, and ideas provided by Licensee — whether
|
|
184
|
+
authored by a human or generated by an AI Agent — are hereby assigned to
|
|
185
|
+
Licensor without restriction or compensation.
|
|
186
|
+
|
|
187
|
+
--------------------------------------------------------------------------------
|
|
188
|
+
6. CONFIDENTIALITY
|
|
189
|
+
--------------------------------------------------------------------------------
|
|
190
|
+
|
|
191
|
+
6.1 Licensee shall hold the Software and all Confidential Information in strict
|
|
192
|
+
confidence, shall not disclose it to any person or AI Agent other than
|
|
193
|
+
Licensee's employees with a need to know who are bound by written
|
|
194
|
+
obligations at least as protective as this Agreement, and shall use it
|
|
195
|
+
solely for the Authorized Purpose. Licensee shall protect it using no less
|
|
196
|
+
than the degree of care it uses for its own most sensitive information, and
|
|
197
|
+
in no event less than a reasonable degree of care. Licensee shall not input,
|
|
198
|
+
expose, or transmit the Software or Confidential Information to any third-
|
|
199
|
+
party AI Agent or service that would acquire rights in, or use for training,
|
|
200
|
+
the material so disclosed.
|
|
201
|
+
|
|
202
|
+
--------------------------------------------------------------------------------
|
|
203
|
+
7. TERM AND TERMINATION
|
|
204
|
+
--------------------------------------------------------------------------------
|
|
205
|
+
|
|
206
|
+
7.1 This Agreement and the license commence on the date Licensee first accesses
|
|
207
|
+
the Software and continue only for the period expressly authorized in
|
|
208
|
+
writing by Licensor (the "Term"). If no period is stated, the Term is
|
|
209
|
+
thirty (30) days.
|
|
210
|
+
|
|
211
|
+
7.2 Licensor may terminate or suspend this Agreement and the license at any
|
|
212
|
+
time, for any reason or no reason, with or without notice, in its sole
|
|
213
|
+
discretion. This license is revocable at will.
|
|
214
|
+
|
|
215
|
+
7.3 This Agreement terminates automatically and immediately upon any breach by
|
|
216
|
+
Licensee, without notice and without opportunity to cure.
|
|
217
|
+
|
|
218
|
+
7.4 Upon any expiration or termination, all rights granted cease immediately,
|
|
219
|
+
and Licensee shall, within five (5) days, cease all use, permanently delete
|
|
220
|
+
or destroy all copies of the Software (including the archival copy, all
|
|
221
|
+
Outputs, and all derivatives and extracts), purge the Software and Outputs
|
|
222
|
+
from any AI Agent context, cache, or store, and certify such destruction in
|
|
223
|
+
writing to Licensor upon request. Sections 1, 3, 4, 5, 6, 7.4, and 8
|
|
224
|
+
through 13 survive termination.
|
|
225
|
+
|
|
226
|
+
--------------------------------------------------------------------------------
|
|
227
|
+
8. AUDIT
|
|
228
|
+
--------------------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
8.1 Licensor may, upon reasonable notice, audit Licensee's use of the Software,
|
|
231
|
+
including by inspecting records and systems and reviewing AI Agent logs and
|
|
232
|
+
access histories, to verify compliance. Licensee shall cooperate. Any
|
|
233
|
+
unauthorized use revealed by an audit constitutes a material breach.
|
|
234
|
+
|
|
235
|
+
--------------------------------------------------------------------------------
|
|
236
|
+
9. DISCLAIMER OF WARRANTIES
|
|
237
|
+
--------------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
9.1 THE SOFTWARE IS PROVIDED "AS IS" AND "AS AVAILABLE," WITH ALL FAULTS AND
|
|
240
|
+
WITHOUT WARRANTY OF ANY KIND. TO THE MAXIMUM EXTENT PERMITTED BY LAW,
|
|
241
|
+
LICENSOR DISCLAIMS ALL WARRANTIES, WHETHER EXPRESS, IMPLIED, STATUTORY, OR
|
|
242
|
+
OTHERWISE, INCLUDING ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR
|
|
243
|
+
A PARTICULAR PURPOSE, TITLE, ACCURACY, AND NON-INFRINGEMENT, AND ANY
|
|
244
|
+
WARRANTY ARISING FROM COURSE OF DEALING OR USAGE OF TRADE. LICENSOR DOES NOT
|
|
245
|
+
WARRANT THAT THE SOFTWARE WILL BE UNINTERRUPTED, ERROR-FREE, OR SECURE, OR
|
|
246
|
+
THAT ANY OUTPUT GENERATED BY THE SOFTWARE OR BY ANY AI AGENT WILL BE
|
|
247
|
+
ACCURATE, COMPLETE, OR FIT FOR ANY PURPOSE.
|
|
248
|
+
|
|
249
|
+
--------------------------------------------------------------------------------
|
|
250
|
+
10. LIMITATION OF LIABILITY
|
|
251
|
+
--------------------------------------------------------------------------------
|
|
252
|
+
|
|
253
|
+
10.1 TO THE MAXIMUM EXTENT PERMITTED BY LAW, LICENSOR SHALL NOT BE LIABLE FOR
|
|
254
|
+
ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, EXEMPLARY, OR PUNITIVE
|
|
255
|
+
DAMAGES, OR FOR ANY LOSS OF PROFITS, REVENUE, DATA, OR GOODWILL, ARISING
|
|
256
|
+
OUT OF OR RELATING TO THIS AGREEMENT, THE SOFTWARE, OR ANY ACTION TAKEN BY
|
|
257
|
+
AN AI AGENT, REGARDLESS OF THE THEORY OF LIABILITY AND EVEN IF ADVISED OF
|
|
258
|
+
THE POSSIBILITY OF SUCH DAMAGES.
|
|
259
|
+
|
|
260
|
+
10.2 LICENSOR'S TOTAL CUMULATIVE LIABILITY ARISING OUT OF OR RELATING TO THIS
|
|
261
|
+
AGREEMENT SHALL NOT EXCEED THE GREATER OF (A) THE FEES ACTUALLY PAID BY
|
|
262
|
+
LICENSEE TO LICENSOR FOR THE SOFTWARE IN THE THREE (3) MONTHS PRECEDING THE
|
|
263
|
+
CLAIM, OR (B) ONE HUNDRED U.S. DOLLARS (US$100).
|
|
264
|
+
|
|
265
|
+
--------------------------------------------------------------------------------
|
|
266
|
+
11. INDEMNIFICATION
|
|
267
|
+
--------------------------------------------------------------------------------
|
|
268
|
+
|
|
269
|
+
11.1 Licensee shall defend, indemnify, and hold harmless Licensor and its
|
|
270
|
+
officers, directors, employees, and agents from and against any and all
|
|
271
|
+
claims, damages, liabilities, costs, and expenses (including reasonable
|
|
272
|
+
attorneys' fees) arising out of or relating to Licensee's use of the
|
|
273
|
+
Software, any use of the Software by any AI Agent acting on Licensee's
|
|
274
|
+
behalf or via Licensee's credentials, or any breach of this Agreement.
|
|
275
|
+
|
|
276
|
+
--------------------------------------------------------------------------------
|
|
277
|
+
12. EXPORT, SANCTIONS, AND COMPLIANCE
|
|
278
|
+
--------------------------------------------------------------------------------
|
|
279
|
+
|
|
280
|
+
12.1 Licensee shall comply with all applicable export control, sanctions, and
|
|
281
|
+
anti-corruption laws and shall not export, re-export, or transfer the
|
|
282
|
+
Software to any prohibited destination, entity, or person, or use it for
|
|
283
|
+
any prohibited end use.
|
|
284
|
+
|
|
285
|
+
--------------------------------------------------------------------------------
|
|
286
|
+
13. GENERAL
|
|
287
|
+
--------------------------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
13.1 Governing Law; Venue. This Agreement is governed by the laws of the
|
|
290
|
+
Commonwealth of Pennsylvania, excluding its conflict-of-laws rules. The
|
|
291
|
+
parties submit to the exclusive jurisdiction of the state and federal
|
|
292
|
+
courts located in Pennsylvania.
|
|
293
|
+
|
|
294
|
+
13.2 Equitable Relief. Licensee acknowledges that any breach of Sections 2, 3,
|
|
295
|
+
4, or 6 would cause irreparable harm for which monetary damages are
|
|
296
|
+
inadequate, and that Licensor is entitled to injunctive relief without the
|
|
297
|
+
need to post bond.
|
|
298
|
+
|
|
299
|
+
13.3 Assignment. Licensee may not assign or transfer this Agreement or any
|
|
300
|
+
rights or obligations, by operation of law or otherwise, without
|
|
301
|
+
Licensor's prior written consent. Any attempted assignment in violation is
|
|
302
|
+
void. Licensor may freely assign.
|
|
303
|
+
|
|
304
|
+
13.4 No Waiver. No failure or delay by Licensor in exercising any right waives
|
|
305
|
+
it. Any waiver must be in a writing signed by Licensor.
|
|
306
|
+
|
|
307
|
+
13.5 Severability. If any provision is held unenforceable, it shall be modified
|
|
308
|
+
to the minimum extent necessary, and the remaining provisions remain in
|
|
309
|
+
full force.
|
|
310
|
+
|
|
311
|
+
13.6 Entire Agreement. Subject to Section 13.8, this Agreement, together
|
|
312
|
+
with any written authorization issued by Licensor, is the entire agreement
|
|
313
|
+
between the parties regarding the Software and supersedes all prior or
|
|
314
|
+
contemporaneous understandings. Any conflicting or additional terms
|
|
315
|
+
proposed by Licensee are rejected.
|
|
316
|
+
|
|
317
|
+
13.7 U.S. Government Rights. If Licensee is a U.S. Government entity, the
|
|
318
|
+
Software is "commercial computer software" and "commercial computer
|
|
319
|
+
software documentation," and any use, duplication, or disclosure is
|
|
320
|
+
subject to the restrictions of this Agreement to the extent permitted by
|
|
321
|
+
applicable Federal Acquisition Regulation and agency supplements.
|
|
322
|
+
|
|
323
|
+
13.8 Amendments; Updated Versions.
|
|
324
|
+
|
|
325
|
+
(a) Licensor may issue an amended or replacement version of this Agreement
|
|
326
|
+
(an "Updated Agreement"). To be effective as to Licensee, an Updated
|
|
327
|
+
Agreement must: (i) conspicuously identify itself as an updated version of
|
|
328
|
+
this Agreement and state its version number and effective date; (ii) be
|
|
329
|
+
provided to Licensee in full by a method reasonably calculated to give
|
|
330
|
+
Licensee notice; and (iii) be affirmatively accepted by Licensee through
|
|
331
|
+
a click-through acceptance, electronic signature, or other written
|
|
332
|
+
acceptance by a person authorized to bind Licensee. Licensor may condition
|
|
333
|
+
any renewal, extension, update, upgrade, or continued access to the
|
|
334
|
+
Software after the end of the then-current Term upon such acceptance.
|
|
335
|
+
|
|
336
|
+
(b) Upon Licensee's affirmative acceptance, the Updated Agreement
|
|
337
|
+
supersedes and replaces all prior versions of this Agreement between
|
|
338
|
+
Licensor and Licensee with respect to the Software, effective on the later
|
|
339
|
+
of the Updated Agreement's stated effective date or Licensee's acceptance
|
|
340
|
+
date. Unless the Updated Agreement expressly states otherwise, any then-
|
|
341
|
+
effective written authorization issued by Licensor remains in force only to
|
|
342
|
+
the extent it is consistent with the Updated Agreement.
|
|
343
|
+
|
|
344
|
+
(c) Until an Updated Agreement becomes effective under this Section 13.8,
|
|
345
|
+
the version of this Agreement previously accepted by Licensee remains
|
|
346
|
+
controlling, subject to Licensor's termination and suspension rights under
|
|
347
|
+
Section 7. Licensor shall retain a reasonably accessible record of each
|
|
348
|
+
version of this Agreement and the date on which Licensee accepted it.
|
|
349
|
+
|
|
350
|
+
================================================================================
|
|
351
|
+
For licensing inquiries, contact: legal@dataerai.com
|
|
352
|
+
Dataerai, Inc. — Center Valley, Pennsylvania, USA
|
|
353
|
+
================================================================================
|
|
354
|
+
|
|
355
|
+
Project-URL: Homepage, https://dataerai.com
|
|
356
|
+
Project-URL: Documentation, https://docs.dataerai.com
|
|
357
|
+
Project-URL: Source, https://github.com/dataerai/dataerai-toolkit
|
|
358
|
+
Keywords: dataerai,data-transfer,sdk,research-data
|
|
359
|
+
Classifier: License :: Other/Proprietary License
|
|
360
|
+
Classifier: Programming Language :: Python :: 3
|
|
361
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
362
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
363
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
364
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
365
|
+
Classifier: Intended Audience :: Science/Research
|
|
366
|
+
Classifier: Operating System :: OS Independent
|
|
367
|
+
Requires-Python: >=3.10
|
|
368
|
+
Description-Content-Type: text/markdown
|
|
369
|
+
License-File: LICENSE
|
|
370
|
+
Provides-Extra: dev
|
|
371
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
372
|
+
Requires-Dist: pytest-mock>=3; extra == "dev"
|
|
373
|
+
Requires-Dist: ipython>=8; extra == "dev"
|
|
374
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
375
|
+
Provides-Extra: rest
|
|
376
|
+
Requires-Dist: httpx>=0.27; extra == "rest"
|
|
377
|
+
Requires-Dist: keyring>=25; extra == "rest"
|
|
378
|
+
Provides-Extra: envelope
|
|
379
|
+
Requires-Dist: cryptography>=42; extra == "envelope"
|
|
380
|
+
Requires-Dist: blake3>=0.3; extra == "envelope"
|
|
381
|
+
Provides-Extra: watermark
|
|
382
|
+
Requires-Dist: cryptography>=42; extra == "watermark"
|
|
383
|
+
Provides-Extra: notebook
|
|
384
|
+
Requires-Dist: ipython>=8; extra == "notebook"
|
|
385
|
+
Provides-Extra: nn-tensorflow
|
|
386
|
+
Requires-Dist: ipython>=8; extra == "nn-tensorflow"
|
|
387
|
+
Requires-Dist: tensorflow<2.22,>=2.21; extra == "nn-tensorflow"
|
|
388
|
+
Requires-Dist: numpy<3,>=2; extra == "nn-tensorflow"
|
|
389
|
+
Requires-Dist: psutil>=5.9; extra == "nn-tensorflow"
|
|
390
|
+
Requires-Dist: nvidia-ml-py>=12; extra == "nn-tensorflow"
|
|
391
|
+
Provides-Extra: nn-pytorch
|
|
392
|
+
Requires-Dist: ipython>=8; extra == "nn-pytorch"
|
|
393
|
+
Requires-Dist: torch>=2.6; extra == "nn-pytorch"
|
|
394
|
+
Requires-Dist: numpy>=1.26; extra == "nn-pytorch"
|
|
395
|
+
Requires-Dist: psutil>=5.9; extra == "nn-pytorch"
|
|
396
|
+
Requires-Dist: nvidia-ml-py>=12; extra == "nn-pytorch"
|
|
397
|
+
Provides-Extra: bambu
|
|
398
|
+
Requires-Dist: dataerai-sdk[rest]; extra == "bambu"
|
|
399
|
+
Requires-Dist: imageio-ffmpeg>=0.6; extra == "bambu"
|
|
400
|
+
Provides-Extra: ml-io
|
|
401
|
+
Requires-Dist: zarr>=3.2; extra == "ml-io"
|
|
402
|
+
Requires-Dist: obstore>=0.5; extra == "ml-io"
|
|
403
|
+
Requires-Dist: numpy>=1.26; extra == "ml-io"
|
|
404
|
+
Requires-Dist: requests>=2.31; extra == "ml-io"
|
|
405
|
+
Requires-Dist: s3fs>=2024.0; extra == "ml-io"
|
|
406
|
+
Provides-Extra: ml
|
|
407
|
+
Requires-Dist: dataerai-sdk[ml-io]; extra == "ml"
|
|
408
|
+
Requires-Dist: torch>=2.4; extra == "ml"
|
|
409
|
+
Requires-Dist: torchdata>=0.9; extra == "ml"
|
|
410
|
+
Requires-Dist: safetensors>=0.4; extra == "ml"
|
|
411
|
+
Provides-Extra: croissant
|
|
412
|
+
Requires-Dist: mlcroissant>=1.0; extra == "croissant"
|
|
413
|
+
Provides-Extra: ml-gpu-cu12
|
|
414
|
+
Requires-Dist: dataerai-sdk[ml]; extra == "ml-gpu-cu12"
|
|
415
|
+
Requires-Dist: kvikio-cu12>=25.10; extra == "ml-gpu-cu12"
|
|
416
|
+
Requires-Dist: cupy-cuda12x>=13; extra == "ml-gpu-cu12"
|
|
417
|
+
Provides-Extra: ml-gpu
|
|
418
|
+
Requires-Dist: dataerai-sdk[ml-gpu-cu12]; extra == "ml-gpu"
|
|
419
|
+
Provides-Extra: serve
|
|
420
|
+
Requires-Dist: onnxruntime>=1.17; extra == "serve"
|
|
421
|
+
Requires-Dist: pillow>=10; extra == "serve"
|
|
422
|
+
Requires-Dist: safetensors>=0.4; extra == "serve"
|
|
423
|
+
Requires-Dist: numpy>=1.26; extra == "serve"
|
|
424
|
+
Requires-Dist: requests>=2.31; extra == "serve"
|
|
425
|
+
Provides-Extra: serve-gpu
|
|
426
|
+
Requires-Dist: onnxruntime-gpu>=1.17; extra == "serve-gpu"
|
|
427
|
+
Requires-Dist: pillow>=10; extra == "serve-gpu"
|
|
428
|
+
Requires-Dist: safetensors>=0.4; extra == "serve-gpu"
|
|
429
|
+
Requires-Dist: numpy>=1.26; extra == "serve-gpu"
|
|
430
|
+
Requires-Dist: requests>=2.31; extra == "serve-gpu"
|
|
431
|
+
Provides-Extra: ml-keras
|
|
432
|
+
Requires-Dist: dataerai-sdk[ml-io]; extra == "ml-keras"
|
|
433
|
+
Requires-Dist: numpy>=2; extra == "ml-keras"
|
|
434
|
+
Requires-Dist: keras>=3; extra == "ml-keras"
|
|
435
|
+
Requires-Dist: pillow>=10; extra == "ml-keras"
|
|
436
|
+
Provides-Extra: ml-hls4ml
|
|
437
|
+
Requires-Dist: dataerai-sdk[ml-keras]; extra == "ml-hls4ml"
|
|
438
|
+
Requires-Dist: hls4ml>=1.3; extra == "ml-hls4ml"
|
|
439
|
+
Provides-Extra: ml-stream
|
|
440
|
+
Requires-Dist: numpy>=2; extra == "ml-stream"
|
|
441
|
+
Requires-Dist: zarr>=3; extra == "ml-stream"
|
|
442
|
+
Provides-Extra: ml-stream-video
|
|
443
|
+
Requires-Dist: dataerai-sdk[ml-stream]; extra == "ml-stream-video"
|
|
444
|
+
Requires-Dist: av>=12; extra == "ml-stream-video"
|
|
445
|
+
Provides-Extra: ml-stream-train
|
|
446
|
+
Requires-Dist: dataerai-sdk[ml-stream]; extra == "ml-stream-train"
|
|
447
|
+
Requires-Dist: torch; extra == "ml-stream-train"
|
|
448
|
+
Provides-Extra: runner
|
|
449
|
+
Requires-Dist: requests>=2; extra == "runner"
|
|
450
|
+
Requires-Dist: nvidia-ml-py>=12.535; extra == "runner"
|
|
451
|
+
Provides-Extra: metaextract
|
|
452
|
+
Requires-Dist: numpy>=1.23; extra == "metaextract"
|
|
453
|
+
Requires-Dist: h5py>=3.7; extra == "metaextract"
|
|
454
|
+
Requires-Dist: igor2>=0.5; extra == "metaextract"
|
|
455
|
+
Requires-Dist: lxml>=4.9; extra == "metaextract"
|
|
456
|
+
Requires-Dist: hyperspy>=2.0; extra == "metaextract"
|
|
457
|
+
Provides-Extra: metaextract-converters
|
|
458
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-converters"
|
|
459
|
+
Requires-Dist: xarray>=0.20; extra == "metaextract-converters"
|
|
460
|
+
Requires-Dist: netCDF4>=1.6; extra == "metaextract-converters"
|
|
461
|
+
Provides-Extra: metaextract-viz
|
|
462
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-viz"
|
|
463
|
+
Requires-Dist: plotly>=5; extra == "metaextract-viz"
|
|
464
|
+
Requires-Dist: nbformat>=4.2; extra == "metaextract-viz"
|
|
465
|
+
Requires-Dist: matplotlib>=3.5; extra == "metaextract-viz"
|
|
466
|
+
Provides-Extra: metaextract-checksums
|
|
467
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-checksums"
|
|
468
|
+
Requires-Dist: blake3>=0.3; extra == "metaextract-checksums"
|
|
469
|
+
Provides-Extra: metaextract-utils
|
|
470
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-utils"
|
|
471
|
+
Requires-Dist: requests>=2.25; extra == "metaextract-utils"
|
|
472
|
+
Provides-Extra: metaextract-containers
|
|
473
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-containers"
|
|
474
|
+
Requires-Dist: netCDF4>=1.6; extra == "metaextract-containers"
|
|
475
|
+
Requires-Dist: scipy>=1.9; extra == "metaextract-containers"
|
|
476
|
+
Requires-Dist: pyarrow>=10; extra == "metaextract-containers"
|
|
477
|
+
Requires-Dist: tifffile>=2022.5; extra == "metaextract-containers"
|
|
478
|
+
Requires-Dist: pillow>=9; extra == "metaextract-containers"
|
|
479
|
+
Provides-Extra: metaextract-microscopy
|
|
480
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-microscopy"
|
|
481
|
+
Requires-Dist: nd2>=0.7; extra == "metaextract-microscopy"
|
|
482
|
+
Requires-Dist: czifile>=2019.7; extra == "metaextract-microscopy"
|
|
483
|
+
Requires-Dist: readlif>=0.6; extra == "metaextract-microscopy"
|
|
484
|
+
Requires-Dist: oiffile>=2021.6; extra == "metaextract-microscopy"
|
|
485
|
+
Provides-Extra: metaextract-medical
|
|
486
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-medical"
|
|
487
|
+
Requires-Dist: pydicom>=2.3; extra == "metaextract-medical"
|
|
488
|
+
Provides-Extra: metaextract-bio
|
|
489
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-bio"
|
|
490
|
+
Requires-Dist: pysam>=0.21; extra == "metaextract-bio"
|
|
491
|
+
Provides-Extra: metaextract-spectroscopy
|
|
492
|
+
Requires-Dist: dataerai-sdk[metaextract]; extra == "metaextract-spectroscopy"
|
|
493
|
+
Requires-Dist: brukeropus>=1.1; extra == "metaextract-spectroscopy"
|
|
494
|
+
Provides-Extra: metaextract-all
|
|
495
|
+
Requires-Dist: dataerai-sdk[metaextract-checksums,metaextract-converters,metaextract-utils,metaextract-viz]; extra == "metaextract-all"
|
|
496
|
+
Requires-Dist: dataerai-sdk[metaextract-containers,metaextract-medical,metaextract-microscopy]; extra == "metaextract-all"
|
|
497
|
+
Requires-Dist: dataerai-sdk[metaextract-bio,metaextract-spectroscopy]; extra == "metaextract-all"
|
|
498
|
+
Dynamic: license-file
|
|
499
|
+
|
|
500
|
+
# Dataerai Python SDK
|
|
501
|
+
|
|
502
|
+
Python clients for the Dataerai transfer daemon and the complete public REST
|
|
503
|
+
API. The daemon client supports authenticated uploads, downloads, metadata, and
|
|
504
|
+
resumable transfers; the optional REST client exposes every public API path.
|
|
505
|
+
|
|
506
|
+
## Requirements
|
|
507
|
+
|
|
508
|
+
- Python ≥ 3.10
|
|
509
|
+
- The `dataerai` binary must be installed and on `PATH` (or pass `binary_path` explicitly)
|
|
510
|
+
- The user must be logged in via `dataerai auth login` before calling `auth_status()` / `upload()` / `download()`
|
|
511
|
+
|
|
512
|
+
## Installation
|
|
513
|
+
|
|
514
|
+
```bash
|
|
515
|
+
pip install --pre "dataerai-sdk>=0.2.0b1,<0.3"
|
|
516
|
+
# or, from source:
|
|
517
|
+
pip install -e sdk/python/
|
|
518
|
+
```
|
|
519
|
+
|
|
520
|
+
Optional ML adapters are installed separately so the base SDK stays small:
|
|
521
|
+
|
|
522
|
+
```bash
|
|
523
|
+
pip install --pre "dataerai-sdk[rest]>=0.2.0b1,<0.3" # Full public REST API
|
|
524
|
+
pip install --pre "dataerai-sdk[ml]>=0.2.0b1,<0.3" # PyTorch Zarr datasets
|
|
525
|
+
pip install --pre "dataerai-sdk[ml-keras]>=0.2.0b1,<0.3" # Keras batches
|
|
526
|
+
pip install --pre "dataerai-sdk[ml-hls4ml]>=0.2.0b1,<0.3" # HLS4ML conversion
|
|
527
|
+
pip install --pre "dataerai-sdk[notebook]>=0.2.0b1,<0.3" # IPython %dataerai magic
|
|
528
|
+
pip install --pre "dataerai-sdk[nn-tensorflow]>=0.2.0b1,<0.3" # TensorFlow provenance
|
|
529
|
+
pip install --pre "dataerai-sdk[envelope]>=0.2.0b1,<0.3" # Encrypted-envelope reader
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
## Full public REST API
|
|
533
|
+
|
|
534
|
+
The daemon client remains the best path for resumable file transfers. Install
|
|
535
|
+
the `rest` extra when a script also needs any public console API endpoint:
|
|
536
|
+
|
|
537
|
+
```python
|
|
538
|
+
from dataerai.rest import RestClient
|
|
539
|
+
|
|
540
|
+
# Reuses credentials written by `dataerai auth login --device`.
|
|
541
|
+
with RestClient() as api:
|
|
542
|
+
response = api.request(
|
|
543
|
+
"GET",
|
|
544
|
+
"/api/projects/",
|
|
545
|
+
params={"page_size": 20},
|
|
546
|
+
)
|
|
547
|
+
response.raise_for_status()
|
|
548
|
+
projects = response.json()
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
`request()` accepts every relative `/api/` path, including endpoints added
|
|
552
|
+
after this SDK version was published, and returns the raw `httpx.Response`.
|
|
553
|
+
`stream()` applies the same security checks while iterating a large response.
|
|
554
|
+
Absolute URLs and redirects are rejected so bearer credentials remain scoped to
|
|
555
|
+
the configured Dataerai server. For headless jobs, set `DATAERAI_SERVER` and
|
|
556
|
+
`DATAERAI_TOKEN`, or provide the CLI-compatible JSON credential object through
|
|
557
|
+
`DATAERAI_CREDENTIALS_JSON`.
|
|
558
|
+
|
|
559
|
+
### Verifiable credentials
|
|
560
|
+
|
|
561
|
+
The REST client provides typed convenience methods without accepting authority
|
|
562
|
+
metadata from callers:
|
|
563
|
+
|
|
564
|
+
```python
|
|
565
|
+
with RestClient() as api:
|
|
566
|
+
issued = api.issue_verifiable_credential("did:dataerai:asset:...")
|
|
567
|
+
api.verifiable_credential_for_did("did:dataerai:asset:...")
|
|
568
|
+
api.verifiable_credential_status(issued["id"])
|
|
569
|
+
api.revoke_verifiable_credential(issued["id"])
|
|
570
|
+
```
|
|
571
|
+
|
|
572
|
+
`issue_sealed_credential_share`, `open_verifiable_credential_share`, and
|
|
573
|
+
`revoke_verifiable_credential_share` cover party-scoped encrypted shares.
|
|
574
|
+
The controller lookup lets a client recover the active credential after a
|
|
575
|
+
reload so permanent revocation remains reachable. Issuance and plaintext presentation require the live
|
|
576
|
+
`verifiable_credentials` organization feature. Verification, status, and
|
|
577
|
+
permanent revocation remain available after policy withdrawal.
|
|
578
|
+
|
|
579
|
+
## Quick start
|
|
580
|
+
|
|
581
|
+
```python
|
|
582
|
+
from dataerai import DataeraiClient
|
|
583
|
+
|
|
584
|
+
with DataeraiClient(binary_path="/usr/local/bin/dataerai") as client:
|
|
585
|
+
# Check auth
|
|
586
|
+
status = client.auth_status()
|
|
587
|
+
print(f"Logged in as {status.user_email} ({status.user_id}), token expires {status.expires_at}")
|
|
588
|
+
if status.user_id is None:
|
|
589
|
+
raise RuntimeError("Upgrade the dataerai CLI to use user-owned uploads")
|
|
590
|
+
|
|
591
|
+
# Upload without needing a project id to hand. owner_id is a UUID, not an
|
|
592
|
+
# email address. Assets are never stored in your personal home area, so the
|
|
593
|
+
# server files this one in your default project ("My Project") and the
|
|
594
|
+
# created asset comes back with owner_type="project".
|
|
595
|
+
result = client.upload(
|
|
596
|
+
"/path/to/data.csv",
|
|
597
|
+
title="My dataset",
|
|
598
|
+
owner_type="user",
|
|
599
|
+
owner_id=status.user_id,
|
|
600
|
+
record_type="dataset",
|
|
601
|
+
on_progress=lambda p: print(f" {p.percent:.0f}% {p.rate_mbps:.1f} MB/s"),
|
|
602
|
+
)
|
|
603
|
+
print(f"Uploaded asset_id={result.asset_id} content_id={result.content_id}")
|
|
604
|
+
|
|
605
|
+
# Download it back
|
|
606
|
+
dl = client.download(result.asset_id, dest_dir="/tmp/downloads")
|
|
607
|
+
for f in dl.files:
|
|
608
|
+
print(f" {f.local_path} ({f.size:,} bytes)")
|
|
609
|
+
|
|
610
|
+
# Read / update metadata
|
|
611
|
+
meta = client.get_metadata(result.asset_id)
|
|
612
|
+
updated = client.set_metadata(
|
|
613
|
+
result.asset_id,
|
|
614
|
+
title="My dataset v2",
|
|
615
|
+
record_type="analysis",
|
|
616
|
+
tags=["csv", "demo"],
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
# Link a derived result to the source asset that produced it
|
|
620
|
+
relationship = client.create_relationship(
|
|
621
|
+
result.asset_id,
|
|
622
|
+
"source-asset-id",
|
|
623
|
+
relationship_type="derived_from",
|
|
624
|
+
analysis_mode="non_destructive",
|
|
625
|
+
qualifiers={"workflow": "clean_svd"},
|
|
626
|
+
)
|
|
627
|
+
related_id = relationship.related_asset["id"] if relationship.related_asset else "source-asset-id"
|
|
628
|
+
print(f"Linked via {relationship.type} to {related_id}")
|
|
629
|
+
```
|
|
630
|
+
|
|
631
|
+
### Feature-controlled Pycroscopy preservation
|
|
632
|
+
|
|
633
|
+
Wrap the daemon client before passing Pycroscopy-labelled uploads and
|
|
634
|
+
relationships through it:
|
|
635
|
+
|
|
636
|
+
```python
|
|
637
|
+
from dataerai.pycroscopy import PycroscopyClient, PycroscopyFeatureAccess
|
|
638
|
+
from dataerai.rest import RestClient
|
|
639
|
+
|
|
640
|
+
with DataeraiClient() as client, RestClient() as rest:
|
|
641
|
+
controlled = PycroscopyClient(
|
|
642
|
+
client,
|
|
643
|
+
PycroscopyFeatureAccess(
|
|
644
|
+
rest,
|
|
645
|
+
owner_type="project",
|
|
646
|
+
owner_id=project_id,
|
|
647
|
+
),
|
|
648
|
+
)
|
|
649
|
+
result = controlled.upload(
|
|
650
|
+
"cleaned_afm.npy",
|
|
651
|
+
title="Cleaned AFM image",
|
|
652
|
+
owner_type="project",
|
|
653
|
+
owner_id=project_id,
|
|
654
|
+
)
|
|
655
|
+
controlled.create_relationship(
|
|
656
|
+
result.asset_id,
|
|
657
|
+
source_asset_id,
|
|
658
|
+
"analysis_of",
|
|
659
|
+
)
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
Install the `rest` extra for `RestClient`. Denied or unavailable decisions stop
|
|
663
|
+
before delegation. Controlled uploads must use the access object's owner, and
|
|
664
|
+
controlled relationship sources must come from uploads through that adapter
|
|
665
|
+
instance. Direct generic SDK operations remain unchanged.
|
|
666
|
+
|
|
667
|
+
## Notebook magic
|
|
668
|
+
|
|
669
|
+
After signing in with the Dataerai CLI, a notebook only needs a destination
|
|
670
|
+
path. The first component names the project; the notebook magic creates that
|
|
671
|
+
project when it is missing, then creates any missing collection components
|
|
672
|
+
beneath it:
|
|
673
|
+
|
|
674
|
+
```python
|
|
675
|
+
DESTINATION_COLLECTION_PATH = "Research / Experiments / July"
|
|
676
|
+
|
|
677
|
+
%load_ext dataerai.magics
|
|
678
|
+
%dataerai $DESTINATION_COLLECTION_PATH
|
|
679
|
+
```
|
|
680
|
+
|
|
681
|
+
The magic publishes a `dataerai_session` variable. Its uploads automatically
|
|
682
|
+
target the selected project and collection:
|
|
683
|
+
|
|
684
|
+
```python
|
|
685
|
+
dataset = dataerai_session.find_asset(
|
|
686
|
+
"did:dataerai:beta:asset:...",
|
|
687
|
+
title="input.csv",
|
|
688
|
+
)
|
|
689
|
+
dataerai_session.download(dataset, "source-data")
|
|
690
|
+
result = dataerai_session.upload(
|
|
691
|
+
"analysis.csv",
|
|
692
|
+
metadata={"component": "derived-output"},
|
|
693
|
+
)
|
|
694
|
+
```
|
|
695
|
+
|
|
696
|
+
Use `%dataerai --as run $DESTINATION_COLLECTION_PATH` to choose a different
|
|
697
|
+
session variable. Regular Python code can call
|
|
698
|
+
`connect_notebook(DESTINATION_COLLECTION_PATH)` instead.
|
|
699
|
+
|
|
700
|
+
### Feature-controlled QICK capture
|
|
701
|
+
|
|
702
|
+
The Dataerai-enabled QICK package accepts any object with `upload()` and
|
|
703
|
+
`create_relationship()`. Pass `dataerai.qick.QickCaptureClient` instead of the
|
|
704
|
+
raw daemon client so the documented QICK path obtains a fresh authenticated,
|
|
705
|
+
owner-scoped organization decision before each mutation:
|
|
706
|
+
|
|
707
|
+
```python
|
|
708
|
+
from dataerai.qick import QickCaptureClient, QickFeatureAccess
|
|
709
|
+
from dataerai.rest import RestClient
|
|
710
|
+
|
|
711
|
+
with DataeraiClient() as client, RestClient() as rest:
|
|
712
|
+
controlled = QickCaptureClient(
|
|
713
|
+
client,
|
|
714
|
+
QickFeatureAccess(rest, owner_type="project", owner_id=project_id),
|
|
715
|
+
)
|
|
716
|
+
result = capture_run(
|
|
717
|
+
controlled,
|
|
718
|
+
cfg,
|
|
719
|
+
acquired_data,
|
|
720
|
+
owner_type="project",
|
|
721
|
+
owner_id=project_id,
|
|
722
|
+
)
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
Install the `rest` extra for `RestClient`. A denied, malformed, unreachable, or
|
|
726
|
+
identity-mismatched decision stops before the underlying SDK call. The REST and
|
|
727
|
+
daemon clients must use the same authenticated person and Dataerai server; the
|
|
728
|
+
adapter verifies both on every protected effect. Direct generic SDK operations
|
|
729
|
+
remain unchanged. Controlled uploads must use the access object's owner, and
|
|
730
|
+
controlled relationship sources must come from uploads through that adapter
|
|
731
|
+
instance.
|
|
732
|
+
|
|
733
|
+
### Trace a notebook run
|
|
734
|
+
|
|
735
|
+
Add `--trace` to capture every subsequent cell's source, timestamps, stdout,
|
|
736
|
+
stderr, structured Python logging records, rich display outputs, returned
|
|
737
|
+
value, and error. Uploads and downloads through the session join the same run
|
|
738
|
+
automatically, following the run-centered provenance pattern used by the QICK
|
|
739
|
+
integration:
|
|
740
|
+
|
|
741
|
+
```python
|
|
742
|
+
%dataerai --trace --notebook analysis.ipynb --title "July analysis" $DESTINATION_COLLECTION_PATH
|
|
743
|
+
|
|
744
|
+
# Run analysis cells and upload their products through dataerai_session.
|
|
745
|
+
result = dataerai_session.upload(
|
|
746
|
+
"analysis.csv",
|
|
747
|
+
record_type="analysis",
|
|
748
|
+
metadata={"component": "derived-output"},
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
%dataerai --finish
|
|
752
|
+
```
|
|
753
|
+
|
|
754
|
+
`%dataerai --finish` waits until its cell completes, then uploads a JSON
|
|
755
|
+
execution-log asset with `record_type="log"`. Every uploaded asset carries a
|
|
756
|
+
shared `notebook_run_id` plus `dataerai-notebook-trace` and
|
|
757
|
+
`notebook-run:<run-id>` tags. The log is linked to each recorded input and
|
|
758
|
+
product with `records_telemetry`, and the published result is available as
|
|
759
|
+
`dataerai_trace`. Run identity is merged onto an existing same-title asset, so
|
|
760
|
+
re-running a fixed notebook or output filename keeps both run tags searchable.
|
|
761
|
+
If a traced cell fails, its source, outputs, error, and traceback are published
|
|
762
|
+
immediately; an explicit `%dataerai --finish` is not required for that failed
|
|
763
|
+
run.
|
|
764
|
+
|
|
765
|
+
Tracing is deliberately opt-in because cell source and output can contain
|
|
766
|
+
secrets or sensitive research data. Review the notebook before enabling it.
|
|
767
|
+
The recorder does not read environment-variable values or scan/upload arbitrary
|
|
768
|
+
files; a file becomes a recorded product only when the notebook uploads it
|
|
769
|
+
through the traced session. Finish an active trace before starting another one
|
|
770
|
+
in the same Python kernel.
|
|
771
|
+
|
|
772
|
+
## TensorFlow neural-network provenance
|
|
773
|
+
|
|
774
|
+
`TensorFlowProvenanceTracker` implements the complete preservation contract
|
|
775
|
+
ported from
|
|
776
|
+
[DataFed_TorchFlow](https://github.com/m3-learning/DataFed_TorchFlow). It saves
|
|
777
|
+
native `.keras` checkpoints containing model and optimizer state, writes a
|
|
778
|
+
machine-readable manifest, captures architecture and hyperparameters, data and
|
|
779
|
+
training-code checksums, outcomes, and full runtime/system properties, and
|
|
780
|
+
authors `derived_from` links to datasets, the notebook, and prior checkpoints.
|
|
781
|
+
Every checkpoint is a first-class `model` asset, so provenance graphs render it
|
|
782
|
+
as a dedicated violet compute node with the Model brain-circuit logo. Final
|
|
783
|
+
checkpoints may also publish hashed figures as `analysis` assets linked to the
|
|
784
|
+
model with `visualizes` relationships. When the notebook session is tracing,
|
|
785
|
+
the tracker automatically reuses its run ID across models, manifests, figures,
|
|
786
|
+
and the execution log.
|
|
787
|
+
|
|
788
|
+
For notebook runs that publish many checkpoints or large analysis artifacts,
|
|
789
|
+
`%dataerai --request-timeout 120 ...` raises the daemon request/response limit
|
|
790
|
+
without changing the separate upload-transfer timeout.
|
|
791
|
+
|
|
792
|
+
```python
|
|
793
|
+
from dataerai.nn import TensorFlowProvenanceTracker
|
|
794
|
+
|
|
795
|
+
tracker = TensorFlowProvenanceTracker(
|
|
796
|
+
model,
|
|
797
|
+
session=dataerai_session,
|
|
798
|
+
run_name="mnist-mlp",
|
|
799
|
+
dataset_references=[
|
|
800
|
+
{
|
|
801
|
+
"asset_id": dataset_asset.asset_id,
|
|
802
|
+
"source_asset_did": SOURCE_ASSET_DID,
|
|
803
|
+
"filename": DATA_PATH.name,
|
|
804
|
+
"sha256": dataset_sha,
|
|
805
|
+
}
|
|
806
|
+
],
|
|
807
|
+
notebook_path=NOTEBOOK_PATH,
|
|
808
|
+
notebook_asset_id=notebook_asset.asset_id,
|
|
809
|
+
)
|
|
810
|
+
history = model.fit(
|
|
811
|
+
x_train,
|
|
812
|
+
y_train,
|
|
813
|
+
epochs=5,
|
|
814
|
+
callbacks=[tracker.callback(training_parameters={"batch_size": 128})],
|
|
815
|
+
)
|
|
816
|
+
final = tracker.save_checkpoint(
|
|
817
|
+
epoch=5,
|
|
818
|
+
label="final",
|
|
819
|
+
record_name="mnist-mlp",
|
|
820
|
+
metrics={"test_accuracy": 0.91},
|
|
821
|
+
training_parameters={"batch_size": 128, "epochs": 5},
|
|
822
|
+
outcomes={"sample_predictions": sample_predictions},
|
|
823
|
+
analysis_artifacts=[
|
|
824
|
+
{
|
|
825
|
+
"path": "artifacts/training-curves.png",
|
|
826
|
+
"kind": "training-curves",
|
|
827
|
+
"caption": "Training and validation metrics across five epochs.",
|
|
828
|
+
"media_type": "image/png",
|
|
829
|
+
}
|
|
830
|
+
],
|
|
831
|
+
)
|
|
832
|
+
```
|
|
833
|
+
|
|
834
|
+
## PyTorch neural-network provenance
|
|
835
|
+
|
|
836
|
+
Install the PyTorch adapter when a training run must preserve native model and
|
|
837
|
+
optimizer state:
|
|
838
|
+
|
|
839
|
+
```bash
|
|
840
|
+
pip install "dataerai-sdk[nn-pytorch]"
|
|
841
|
+
```
|
|
842
|
+
|
|
843
|
+
`PyTorchProvenanceTracker` emits the same DataFed_TorchFlow preservation
|
|
844
|
+
concepts as the TensorFlow tracker while keeping framework-native checkpoint
|
|
845
|
+
validation inside the adapter. Each `.pt` file contains only the versioned
|
|
846
|
+
schema, run/epoch identity, model `state_dict`, and optimizer `state_dict`.
|
|
847
|
+
Before any upload, the tracker reloads the file on CPU with
|
|
848
|
+
`weights_only=True`; a checkpoint that cannot pass that restricted loader is
|
|
849
|
+
rejected.
|
|
850
|
+
|
|
851
|
+
```python
|
|
852
|
+
import torch
|
|
853
|
+
|
|
854
|
+
from dataerai.nn import PyTorchProvenanceTracker
|
|
855
|
+
|
|
856
|
+
model = MyModel()
|
|
857
|
+
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
|
|
858
|
+
tracker = PyTorchProvenanceTracker(
|
|
859
|
+
model,
|
|
860
|
+
optimizer,
|
|
861
|
+
session=dataerai_session,
|
|
862
|
+
run_name="mnist-cnn",
|
|
863
|
+
dataset_references=[
|
|
864
|
+
{
|
|
865
|
+
"asset_id": dataset_asset.asset_id,
|
|
866
|
+
"source_asset_did": SOURCE_ASSET_DID,
|
|
867
|
+
"filename": DATA_PATH.name,
|
|
868
|
+
"sha256": dataset_sha,
|
|
869
|
+
}
|
|
870
|
+
],
|
|
871
|
+
notebook_path=NOTEBOOK_PATH,
|
|
872
|
+
notebook_asset_id=notebook_asset.asset_id,
|
|
873
|
+
)
|
|
874
|
+
|
|
875
|
+
for epoch in range(1, 6):
|
|
876
|
+
train_one_epoch(model, optimizer)
|
|
877
|
+
tracker.save_checkpoint(
|
|
878
|
+
epoch=epoch,
|
|
879
|
+
label="epoch",
|
|
880
|
+
metrics={"loss": current_loss},
|
|
881
|
+
training_parameters={"learning_rate": 1e-3, "epochs": 5},
|
|
882
|
+
)
|
|
883
|
+
```
|
|
884
|
+
|
|
885
|
+
With a `NotebookSession`, checkpoints are published as `model` assets.
|
|
886
|
+
Manifests and optional analysis figures are ordinary `analysis` assets.
|
|
887
|
+
Qualified `derived_from`, `describes`, and `visualizes` relationships connect
|
|
888
|
+
them to datasets, the training notebook, and the preceding checkpoint. Without
|
|
889
|
+
a session, the identical checkpoint and manifest contracts are written locally.
|
|
890
|
+
|
|
891
|
+
System capture intentionally does not enumerate environment variables. It
|
|
892
|
+
records OS/host, CPU and memory properties, framework-visible devices, optional
|
|
893
|
+
NVML GPU memory, Python/framework/package versions, and only a small allowlist
|
|
894
|
+
of determinism and thread settings.
|
|
895
|
+
|
|
896
|
+
## API reference
|
|
897
|
+
|
|
898
|
+
### Encrypted envelopes
|
|
899
|
+
|
|
900
|
+
The optional reader keeps OAuth credentials in the Go daemon. Python receives
|
|
901
|
+
only the audited per-envelope DEK over the owner-only local IPC endpoint:
|
|
902
|
+
|
|
903
|
+
```python
|
|
904
|
+
from dataerai import envelope
|
|
905
|
+
|
|
906
|
+
with envelope.open("data.denv") as env:
|
|
907
|
+
print(env.audit)
|
|
908
|
+
env.verify_signature() # server-bound cache; offline while fresh (5 minutes)
|
|
909
|
+
env.unlock() # daemon IPC; audited purpose="open"
|
|
910
|
+
print(env.namelist())
|
|
911
|
+
with env.open("member.csv") as member:
|
|
912
|
+
first_kib = member.read(1024)
|
|
913
|
+
env.extractall("out") # atomic; failures leave no partial output
|
|
914
|
+
```
|
|
915
|
+
|
|
916
|
+
Populate or refresh the public-key cache with
|
|
917
|
+
`dataerai envelope verify data.denv`. A new Python process must call `unlock()`
|
|
918
|
+
again; neither the SDK nor daemon persists the DEK.
|
|
919
|
+
|
|
920
|
+
The whole path — parsing, fresh-cache verification, `unlock()` and
|
|
921
|
+
`extractall()` — is portable. The daemon transport is an `AF_UNIX` socket on
|
|
922
|
+
POSIX and a named pipe on Windows, so no platform needs to fall back to the Go
|
|
923
|
+
CLI.
|
|
924
|
+
|
|
925
|
+
### `DataeraiClient(*, socket_path, binary_path, auto_start, start_timeout_s, request_timeout_s)`
|
|
926
|
+
|
|
927
|
+
| Parameter | Default | Description |
|
|
928
|
+
|---|---|---|
|
|
929
|
+
| `socket_path` | `$DATAERAI_SOCKET`, otherwise the shared platform default described below | Daemon IPC endpoint: an `AF_UNIX` socket on POSIX, a named pipe on Windows |
|
|
930
|
+
| `binary_path` | `None` | Path to the `dataerai` binary (required for `auto_start`) |
|
|
931
|
+
| `auto_start` | `True` | Spawn the daemon if no daemon is listening on the endpoint |
|
|
932
|
+
| `start_timeout_s` | `10.0` | Seconds to wait for the daemon endpoint to become available |
|
|
933
|
+
| `request_timeout_s` | `30.0` | Per-request timeout in seconds |
|
|
934
|
+
|
|
935
|
+
Use as a context manager (`with DataeraiClient(...) as client:`) for automatic cleanup, or call `client.connect()` / `client.close()` manually.
|
|
936
|
+
|
|
937
|
+
A daemon spawned by this client must remain running through readiness and
|
|
938
|
+
transport connection. An early exit raises `RuntimeError` before starting the
|
|
939
|
+
reader or sending commands; a transport opened during that exit is closed.
|
|
940
|
+
This prevents an unrelated listener from turning a failed spawn into a
|
|
941
|
+
successful connection. Resolve the startup failure and create a fresh client.
|
|
942
|
+
Calls that reuse an already-running daemon retain their existing behavior.
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
### Daemon endpoint selection
|
|
946
|
+
|
|
947
|
+
On POSIX, the daemon and both SDKs choose the first private directory among
|
|
948
|
+
`XDG_RUNTIME_DIR`, `/run/user/<uid>`, and the platform temporary directory.
|
|
949
|
+
A private directory must be a real directory owned by the current user with
|
|
950
|
+
no group or other permission bits. If none qualifies, they use
|
|
951
|
+
`~/.dataerai/run/dataerai-transfer.sock`, creating missing fallback directories
|
|
952
|
+
with mode `0700`. The socket name remains `dataerai-transfer.sock`.
|
|
953
|
+
|
|
954
|
+
`DATAERAI_SOCKET` overrides this selection. An explicit endpoint must be an
|
|
955
|
+
absolute path without `.` or `..` components. Create its parent directory
|
|
956
|
+
first; arbitrary custom parents are not created. Before connecting or removing
|
|
957
|
+
a stale socket, clients verify that it is a real socket owned by the current
|
|
958
|
+
user and that its directory and ancestors cannot be replaced by another user.
|
|
959
|
+
An explicit socket may use a root-owned sticky directory. A symlink at the
|
|
960
|
+
socket or its immediate parent is refused; safe system symlinks higher in the
|
|
961
|
+
path are supported. On macOS, use a private child directory beneath `/tmp`
|
|
962
|
+
or the physical directory path instead of `/tmp` as the immediate parent.
|
|
963
|
+
|
|
964
|
+
An ownership refusal reports `DaemonEndpointError` in the SDKs or
|
|
965
|
+
`ErrEndpointNotOwned` in Go and names the offending path. Point
|
|
966
|
+
`DATAERAI_SOCKET` at an endpoint in a private directory to resolve it. The
|
|
967
|
+
clients preserve refused entries and do not silently select another endpoint.
|
|
968
|
+
Existing fallback directories are checked, never repaired with `chmod`.
|
|
969
|
+
|
|
970
|
+
On Linux without a private runtime directory, an old daemon may still be
|
|
971
|
+
listening under shared `/tmp` while upgraded clients choose the home fallback.
|
|
972
|
+
Finish active transfers and restart the daemon with matching client versions.
|
|
973
|
+
The old process and socket are not moved automatically. Windows keeps
|
|
974
|
+
`\\.\pipe\dataerai-transfer-<sanitized USERNAME>`; unsupported username
|
|
975
|
+
characters map to one underscore per Unicode code point in all three clients.
|
|
976
|
+
|
|
977
|
+
### Windows pipe identity
|
|
978
|
+
|
|
979
|
+
The Go CLI and Python SDK open only local named pipes and verify the server
|
|
980
|
+
process belongs to the same Windows account before sending application data.
|
|
981
|
+
Python opens with anonymous SQOS as well as overlapped I/O; Go uses the existing
|
|
982
|
+
anonymous go-winio dial. If Windows cannot report the server identity or the
|
|
983
|
+
account differs, the connection is closed and a typed endpoint refusal is
|
|
984
|
+
returned. Missing pipes remain a readiness miss and busy pipes retain bounded
|
|
985
|
+
retry behavior.
|
|
986
|
+
|
|
987
|
+
Restart the expected daemon under your own account if verification fails.
|
|
988
|
+
Do not weaken the pipe permissions or connect to a pipe on another machine.
|
|
989
|
+
Node/Electron also requires the proof exchange described below.
|
|
990
|
+
|
|
991
|
+
### Daemon proof and key storage
|
|
992
|
+
|
|
993
|
+
Before returning a connection or sending application commands, Go Dial and
|
|
994
|
+
both SDKs send `ipc.hello` and verify the daemon's proof against a private
|
|
995
|
+
32-byte key. A failed proof, invalid key permissions or unavailable identity
|
|
996
|
+
check closes the connection with an endpoint error. Requests that time out or
|
|
997
|
+
are cancelled while Node is waiting for proof are removed before the queue is
|
|
998
|
+
released. A spawned daemon must remain alive through this exchange.
|
|
999
|
+
|
|
1000
|
+
The daemon creates the key once at the platform configuration location:
|
|
1001
|
+
|
|
1002
|
+
| Platform | Key file |
|
|
1003
|
+
| --- | --- |
|
|
1004
|
+
| Linux | `$XDG_CONFIG_HOME/dataerai/ipc-key`, or `~/.config/dataerai/ipc-key` |
|
|
1005
|
+
| macOS | `~/Library/Application Support/dataerai/ipc-key` |
|
|
1006
|
+
| Windows | `%APPDATA%\dataerai\ipc-key` |
|
|
1007
|
+
|
|
1008
|
+
Run the daemon and SDK under the same account and configuration environment.
|
|
1009
|
+
The SDKs only read the key. They never create or repair it. POSIX files must be
|
|
1010
|
+
owned by the current user, exactly 32 bytes and mode `0600` or `0400`, with safe
|
|
1011
|
+
ACLs and parent directories. Windows checks the owner, ACL and path handles;
|
|
1012
|
+
Node requires Windows PowerShell 5.1 in FullLanguage mode for that key check.
|
|
1013
|
+
A missing key is distinct from an unsafe existing key.
|
|
1014
|
+
|
|
1015
|
+
For version compatibility, only the daemon's matching `ERR_UNKNOWN_METHOD`
|
|
1016
|
+
reply permits legacy use. Go/Python require successful OS identity checks;
|
|
1017
|
+
Node permits this on POSIX after endpoint ownership checks. Node on Windows
|
|
1018
|
+
refuses legacy daemons unless `DATAERAI_IPC_ALLOW_LEGACY_DAEMON=1` is explicitly
|
|
1019
|
+
set. Allowed legacy use logs once per process. Upgrade and restart the daemon
|
|
1020
|
+
to use the proof protocol; this setting never bypasses an invalid key file or
|
|
1021
|
+
an incorrect proof. Desktop releases must bundle a matching daemon and SDK.
|
|
1022
|
+
|
|
1023
|
+
If verification fails, check that the expected daemon is running as you and
|
|
1024
|
+
that both processes use the same configuration directory. Finish active
|
|
1025
|
+
transfers before restarting it. The key is not rotated or deleted by logout.
|
|
1026
|
+
After deliberately replacing a key, restart every daemon that loaded it; a
|
|
1027
|
+
running daemon retains its original key. Do not make the key readable by other
|
|
1028
|
+
users or copy it into logs or support tickets.
|
|
1029
|
+
|
|
1030
|
+
Node cannot request anonymous Windows pipe impersonation at the OS level.
|
|
1031
|
+
The proof prevents application credentials reaching an unproven server, but
|
|
1032
|
+
does not remove the OS-token impersonation risk from a malicious pipe server
|
|
1033
|
+
with impersonation privileges. Native Windows release verification remains
|
|
1034
|
+
required.
|
|
1035
|
+
|
|
1036
|
+
### Console URL requirements
|
|
1037
|
+
|
|
1038
|
+
Daemon sign-in and token refresh require an HTTPS origin. Loopback HTTP is
|
|
1039
|
+
allowed for local development. Remote HTTP requires explicitly setting
|
|
1040
|
+
`DATAERAI_ALLOW_INSECURE_HTTP=1` in the daemon's environment. CLI sign-in also
|
|
1041
|
+
supports `--insecure-http` for that login only; the flag does not authorize
|
|
1042
|
+
future reads or refreshes of stored HTTP credentials. URL userinfo, path
|
|
1043
|
+
prefixes, queries and fragments are refused. Token endpoints must respond
|
|
1044
|
+
directly, because credential-bearing requests do not follow redirects.
|
|
1045
|
+
|
|
1046
|
+
Loopback OAuth requests — including `localhost` and `*.localhost` — bypass
|
|
1047
|
+
`HTTP_PROXY` and `HTTPS_PROXY`, so cleartext that is permitted for being local
|
|
1048
|
+
is not dialled through a proxy. Remote origins keep the daemon environment's
|
|
1049
|
+
proxy settings.
|
|
1050
|
+
|
|
1051
|
+
### Methods
|
|
1052
|
+
|
|
1053
|
+
| Method | Returns | Description |
|
|
1054
|
+
|---|---|---|
|
|
1055
|
+
| `auth_status()` | `AuthStatus` | Logged-in user ID, email, and token expiry; `user_id` is `None` with older daemons |
|
|
1056
|
+
| `list_tree()` | `list[TreeNode]` | List visible workspaces, projects, and collections |
|
|
1057
|
+
| `list_projects()` | `list[Project]` | Projects from the tree (see the write-access caveat below) |
|
|
1058
|
+
| `list_collections(*, owner_type=None, owner_id=None)` | `list[Collection]` | Collections from the tree, flattened and filterable |
|
|
1059
|
+
| `list_collection_assets(collection_id, ...)` | `AssetSearchPage` | One page of a collection's assets |
|
|
1060
|
+
| `get_collection_manifest(collection_id)` | `CollectionManifest` | Every **file** in a collection, with which ones a sync will not deliver |
|
|
1061
|
+
| `find_collection_assets(collection_id, ...)` | `list[AssetSummary]` | Every asset in a collection, paging until exhausted |
|
|
1062
|
+
| `create_collection(title, *, owner_type, owner_id, parent_id=None)` | `Collection` | Create a collection |
|
|
1063
|
+
| `ensure_collection_path(path, *, create_project=False, project_description="")` | `CollectionDestination` | Resolve `Project / Collection / ...`, creating missing collection components and optionally the project |
|
|
1064
|
+
| `search_assets(query, ...)` | `AssetSearchPage` | Search one page of visible assets |
|
|
1065
|
+
| `find_assets(query, ...)` | `list[AssetSummary]` | Search and collect every result page |
|
|
1066
|
+
| `create_project(name, *, description="")` | `Project` | Create a project (+ root collection, owner membership, default allocation); requires write scope |
|
|
1067
|
+
| `list_allocations(owner_id, *, owner_type="project")` | `list[Allocation]` | A project's storage allocations, with quota headroom |
|
|
1068
|
+
| `upload(local_path, *, title, owner_type, owner_id, record_type=None, ...)` | `UploadResult` | Upload a file with an optional server-validated record type; blocks until complete |
|
|
1069
|
+
| `upload(local_path, *, title, owner_type, owner_id, record_type=None, ...)` | `UploadResult` | Upload one file — or a sequence of paths as **one multi-file asset**; blocks until complete |
|
|
1070
|
+
| `upload_many(items, *, owner_type, owner_id, concurrency=4, ...)` | `BulkUploadResult` | Upload many assets concurrently, each with its own metadata; reports partial failure |
|
|
1071
|
+
| `update_content(asset_id, local_path, ...)` | `UploadResult` | Attach a **new content version** to an existing asset, by ID; blocks until complete |
|
|
1072
|
+
| `list_content_versions(asset_id)` | `list[ContentVersion]` | List an asset's content versions (in-flight ones first, then available newest-first) |
|
|
1073
|
+
| `download(asset_id, dest_dir, ...)` | `DownloadResult` | Download latest asset content; blocks until complete |
|
|
1074
|
+
| `download_collection(collection_id, dest_dir, ...)` | `CollectionDownloadResult` | Download every asset in a collection recursively, preserving sub-collections as sub-directories; blocks until all transfers complete |
|
|
1075
|
+
| `list_transfers()` | `list[TransferSummary]` | Transfers the local daemon is tracking |
|
|
1076
|
+
| `list_server_transfers(...)` | `ServerTransferPage` | Transfers the console records, including ones started elsewhere |
|
|
1077
|
+
| `cancel_transfer(transfer_id)` | `str` | Cancel a transfer; waits for the daemon to confirm |
|
|
1078
|
+
| `pause_transfer(transfer_id)` | `None` | Pause a transfer (**fire-and-forget**) |
|
|
1079
|
+
| `resume_transfer(transfer_id, credentials=None)` | `None` | Resume a paused transfer (**fire-and-forget**) |
|
|
1080
|
+
| `shutdown_daemon()` | `None` | Stop the daemon **for every client on the machine**, then close this client (**fire-and-forget**) |
|
|
1081
|
+
| `get_metadata(asset_id)` | `AssetMetadata` | Retrieve asset metadata |
|
|
1082
|
+
| `set_metadata(asset_id, **fields)` | `AssetMetadata` | Update metadata fields, including `record_type` |
|
|
1083
|
+
| `set_metadata_many(updates, *, concurrency=4)` | `BulkMetadataResult` | Apply a per-asset metadata edit to many assets; reports partial failure |
|
|
1084
|
+
| `create_relationship(from_asset_id, to_asset_id, rel_type=None, *, relationship_type=None, ...)` | `Relationship` | Create a directed provenance link between two assets |
|
|
1085
|
+
| `on(event, handler=None)` | handler / decorator | Subscribe to a daemon event, or to `"*"` for all (see [Daemon events](#daemon-events)) |
|
|
1086
|
+
| `off(event, handler)` | `bool` | Unsubscribe; `False` if it was not registered |
|
|
1087
|
+
| `sync_collection(collection_id, dest_dir, *, prune=False, dry_run=False, ...)` | `SyncResult` | Incrementally sync a collection into a local folder; **`prune` deletes local files** |
|
|
1088
|
+
| `delete_relationship(from_asset_id, relationship_id)` | `None` | Delete an edge from its **source** asset — the inverse of `create_relationship()` |
|
|
1089
|
+
|
|
1090
|
+
### Choosing an allocation
|
|
1091
|
+
|
|
1092
|
+
`upload()` and `update_content()` accept an `allocation_id`; this is how you get
|
|
1093
|
+
one:
|
|
1094
|
+
|
|
1095
|
+
```python
|
|
1096
|
+
for a in client.list_allocations(project_id):
|
|
1097
|
+
print(a.allocation_id, a.alias, a.repository_name,
|
|
1098
|
+
f"{a.bytes_free:,} B free", f"{a.records_free:,} records free",
|
|
1099
|
+
"FULL" if a.is_full else "")
|
|
1100
|
+
```
|
|
1101
|
+
|
|
1102
|
+
Checking before you upload is worth the round trip. A full allocation doesn't
|
|
1103
|
+
fail at the point of cause — uploads are rejected, and a rejected upload can
|
|
1104
|
+
leave a contentless asset that surfaces much later as "no downloadable
|
|
1105
|
+
content".
|
|
1106
|
+
|
|
1107
|
+
> **`is_full` is necessary, not sufficient.** `True` means an upload will be
|
|
1108
|
+
> rejected. `False` does *not* guarantee one succeeds: the console also
|
|
1109
|
+
> hard-blocks on billing suspension, enforces a temporary grace ceiling instead
|
|
1110
|
+
> of `data_volume_limit` during a grace window, and walks up parent
|
|
1111
|
+
> allocations — none of which the daemon reports, so none are visible here.
|
|
1112
|
+
> Read `False` as "the reported quota has room".
|
|
1113
|
+
|
|
1114
|
+
A limit of `0` means zero capacity, not unlimited — the console's gate is a
|
|
1115
|
+
plain `current + incoming > limit` with no special case for zero. A missing limit
|
|
1116
|
+
likewise reads as full rather than unlimited: fail-closed is the safe
|
|
1117
|
+
direction, because wrongly reporting room is the failure this is meant to
|
|
1118
|
+
prevent.
|
|
1119
|
+
|
|
1120
|
+
> **An empty list does not mean uploads will fail.** The server resolves an
|
|
1121
|
+
> upload's allocation through a cascade — the owner's allocation on its
|
|
1122
|
+
> preferred repository, else the owner's default, else, for a project-owned
|
|
1123
|
+
> asset, one held by the project's *owning user*. That last step is invisible
|
|
1124
|
+
> here, because the daemon lists only what the project holds directly. So an
|
|
1125
|
+
> upload with no `allocation_id` may land in an allocation that is not in this
|
|
1126
|
+
> list; passing one from here pins the destination instead.
|
|
1127
|
+
### Listing what you have
|
|
1128
|
+
|
|
1129
|
+
`list_collection_assets()` / `find_collection_assets()` enumerate a collection's
|
|
1130
|
+
contents without downloading them — the alternative before they existed was
|
|
1131
|
+
`download_collection()`, which answers the question by fetching everything:
|
|
1132
|
+
|
|
1133
|
+
```python
|
|
1134
|
+
for asset in client.find_collection_assets(collection_id):
|
|
1135
|
+
print(asset.asset_id, asset.title, asset.size_bytes, asset.has_content)
|
|
1136
|
+
```
|
|
1137
|
+
|
|
1138
|
+
`find_collection_assets()` pages until exhausted, and raises rather than looping
|
|
1139
|
+
if the server claims another page without a cursor or repeats one.
|
|
1140
|
+
|
|
1141
|
+
> **`list_projects()` and `list_collections()` show what you can upload into,
|
|
1142
|
+
> not everything you can read.** Both are projections of `list_tree()`, and the
|
|
1143
|
+
> daemon omits any project you lack `WRITE_METADATA` on — so a project you can
|
|
1144
|
+
> read but not write to is absent, and so are its collections. There is no
|
|
1145
|
+
> SDK call that lists read-only projects today.
|
|
1146
|
+
|
|
1147
|
+
`list_projects()` populates only `project_id`, `name` and `root_collection_id`;
|
|
1148
|
+
`tree.list` carries nothing else, so the remaining `Project` fields are `None`.
|
|
1149
|
+
Collections shared with you directly are attached to the personal node, so one
|
|
1150
|
+
returned under `owner_type="user"` may carry another project's
|
|
1151
|
+
`owner_project_id`.
|
|
1152
|
+
|
|
1153
|
+
For headless notebooks, pass `record_type` to `upload()` or `set_metadata()`.
|
|
1154
|
+
Both operations use the authenticated transfer daemon, so integrations do not
|
|
1155
|
+
need to read the CLI credential store or expose access tokens to Python.
|
|
1156
|
+
|
|
1157
|
+
### What a collection actually contains
|
|
1158
|
+
|
|
1159
|
+
`list_collection_assets()` enumerates *assets*. `get_collection_manifest()`
|
|
1160
|
+
enumerates the **files** inside them, flattened, with each one's position in the
|
|
1161
|
+
collection tree — and, crucially, which of them a download or sync will
|
|
1162
|
+
**silently not deliver**:
|
|
1163
|
+
|
|
1164
|
+
```python
|
|
1165
|
+
manifest = client.get_collection_manifest(collection_id)
|
|
1166
|
+
print(f"{manifest.total_files} files, {manifest.total_bytes:,} bytes")
|
|
1167
|
+
|
|
1168
|
+
for entry in manifest.undownloadable:
|
|
1169
|
+
why = "external" if entry.external else "no checksum on the server"
|
|
1170
|
+
print(f" will NOT arrive: {entry.relative_path} ({why})")
|
|
1171
|
+
```
|
|
1172
|
+
|
|
1173
|
+
This is the answer to *"why does my synced folder have fewer files than the
|
|
1174
|
+
collection?"*. A sync omits two kinds of entry and reports neither per-file:
|
|
1175
|
+
**external** entries, whose bytes live outside Dataerai, and entries the server
|
|
1176
|
+
holds **no checksum** for, which the daemon refuses to write because it cannot
|
|
1177
|
+
verify them (the `skipped_unverified` count). The daemon collapses both into one
|
|
1178
|
+
`is_downloadable` flag, so `external` is what tells them apart.
|
|
1179
|
+
|
|
1180
|
+
The manifest is **complete or it raises** — the daemon assembles every page
|
|
1181
|
+
itself and rejects a result whose totals or snapshot shifted while paging, which
|
|
1182
|
+
is what makes it safe to use as the expectation you check a sync against. Note
|
|
1183
|
+
it carries **no checksums**; the checksum is consumed to compute
|
|
1184
|
+
`is_downloadable`, not forwarded.
|
|
1185
|
+
|
|
1186
|
+
`collection_id` must be a **canonical** UUID here — lower-case, hyphenated, no
|
|
1187
|
+
braces or `urn:` prefix. The daemon rejects other spellings rather than
|
|
1188
|
+
normalising them.
|
|
1189
|
+
|
|
1190
|
+
`download_collection()` raises on a failed or timed-out transfer, but an asset
|
|
1191
|
+
the daemon could never queue is *reported* rather than raised: its ID lands in
|
|
1192
|
+
`result.failed_assets`, and it is absent from both `result.transfers` and
|
|
1193
|
+
`result.asset_count`. Check that field before treating the download as complete.
|
|
1194
|
+
|
|
1195
|
+
```python
|
|
1196
|
+
result = client.download_collection(collection_id, "/data/out")
|
|
1197
|
+
if result.failed_assets:
|
|
1198
|
+
raise RuntimeError(f"{len(result.failed_assets)} assets did not download: "
|
|
1199
|
+
f"{result.failed_assets}")
|
|
1200
|
+
```
|
|
1201
|
+
|
|
1202
|
+
### Watching and controlling transfers
|
|
1203
|
+
|
|
1204
|
+
The daemon runs a real job engine behind `upload()` and `download()`: chunked
|
|
1205
|
+
multipart, parts in parallel, resume-from-partial across restarts, and at most
|
|
1206
|
+
four transfers at once with the rest queued. These expose it:
|
|
1207
|
+
|
|
1208
|
+
```python
|
|
1209
|
+
for t in client.list_transfers():
|
|
1210
|
+
print(t.transfer_id, t.status, t.direction, f"{t.percent:.0f}%")
|
|
1211
|
+
|
|
1212
|
+
# Transfers started anywhere — the web app, another device, a runner
|
|
1213
|
+
page = client.list_server_transfers(type="upload")
|
|
1214
|
+
print(page.count, "total;", [t.executor for t in page.transfers])
|
|
1215
|
+
```
|
|
1216
|
+
|
|
1217
|
+
> **`pause_transfer()` and `resume_transfer()` are fire-and-forget, and
|
|
1218
|
+
> therefore silent on failure.** The daemon writes nothing on success, so they
|
|
1219
|
+
> cannot wait for an acknowledgement — and one naming an unknown transfer is
|
|
1220
|
+
> discarded without raising. This mirrors the Node SDK rather than inventing a
|
|
1221
|
+
> Python-only contract. Confirmation is asynchronous: the worker emits
|
|
1222
|
+
> `transfer.paused`, and `list_transfers()` then shows `paused`.
|
|
1223
|
+
>
|
|
1224
|
+
> `cancel_transfer()` is different — the daemon acks it, so it waits and raises
|
|
1225
|
+
> `ERR_TRANSFER_NOT_FOUND` if the transfer is unknown.
|
|
1226
|
+
|
|
1227
|
+
### Stopping the daemon
|
|
1228
|
+
|
|
1229
|
+
`shutdown_daemon()` completes the set of fire-and-forget commands the Node SDK
|
|
1230
|
+
sends. The daemon logs the request, signals its own shutdown and closes the
|
|
1231
|
+
connection without replying.
|
|
1232
|
+
|
|
1233
|
+
> ⚠️ **The daemon is shared machine-wide.** Stopping it aborts in-flight
|
|
1234
|
+
> transfers for *every* client — the desktop app and any running CLI included.
|
|
1235
|
+
> Use `close()` unless you specifically mean to stop the daemon, e.g. tearing
|
|
1236
|
+
> down an ephemeral environment or a test fixture.
|
|
1237
|
+
|
|
1238
|
+
The client is closed afterwards: one left open against a stopped daemon fails
|
|
1239
|
+
every later call with a connection error that says nothing about the cause.
|
|
1240
|
+
Build a new client to reconnect — one with `auto_start` and a `binary_path`
|
|
1241
|
+
respawns the daemon.
|
|
1242
|
+
|
|
1243
|
+
**What a blocking call sees.** Cancelling a transfer that an `upload()` or
|
|
1244
|
+
`download()` is waiting on makes *that* call raise `DaemonError` with
|
|
1245
|
+
`code == "cancelled"` — the daemon emits `transfer.failed` with that code
|
|
1246
|
+
rather than a `transfer.cancelled` message, so check the code rather than
|
|
1247
|
+
expecting a clean return. **Pausing is different and easier to get wrong:** a
|
|
1248
|
+
paused transfer sends no terminal event, so a blocked `upload()` keeps waiting
|
|
1249
|
+
and eventually raises `DaemonTimeoutError` after its `transfer_timeout_s`.
|
|
1250
|
+
Pausing does not extend that budget.
|
|
1251
|
+
|
|
1252
|
+
Cancelling an upload that is still running needs its transfer id, and
|
|
1253
|
+
`upload()` does not return until it finishes — so capture the id from a
|
|
1254
|
+
progress event:
|
|
1255
|
+
|
|
1256
|
+
```python
|
|
1257
|
+
seen = {}
|
|
1258
|
+
client.upload(path, ..., on_progress=lambda e: seen.setdefault("id", e.transfer_id))
|
|
1259
|
+
# ... from another thread, once seen has an id:
|
|
1260
|
+
client.cancel_transfer(seen["id"])
|
|
1261
|
+
```
|
|
1262
|
+
|
|
1263
|
+
### Keeping a local folder in sync
|
|
1264
|
+
|
|
1265
|
+
`download_collection()` fetches everything, every time. `sync_collection()`
|
|
1266
|
+
compares the collection against what is already on disk and moves only what
|
|
1267
|
+
changed, keeping state under the destination so it works across runs:
|
|
1268
|
+
|
|
1269
|
+
```python
|
|
1270
|
+
result = client.sync_collection(
|
|
1271
|
+
collection_id,
|
|
1272
|
+
"/data/my-collection",
|
|
1273
|
+
on_progress=lambda p: print(f" {p.percent:.0f}% {p.path}"),
|
|
1274
|
+
)
|
|
1275
|
+
print(f"{result.downloaded} downloaded, {result.pruned} removed, "
|
|
1276
|
+
f"{result.elapsed_ms} ms")
|
|
1277
|
+
|
|
1278
|
+
if not result.is_complete:
|
|
1279
|
+
print(
|
|
1280
|
+
f"{result.failed} files could not be fetched; "
|
|
1281
|
+
f"{result.conflicts} local conflicts were preserved"
|
|
1282
|
+
)
|
|
1283
|
+
if result.skipped_unverified:
|
|
1284
|
+
print(f"{result.skipped_unverified} withheld — the server had no checksum")
|
|
1285
|
+
```
|
|
1286
|
+
|
|
1287
|
+
> ⚠️ **`prune=True` deletes local files.** Anything under the destination the
|
|
1288
|
+
> collection no longer contains is removed. Preview it first — `dry_run=True`
|
|
1289
|
+
> computes the same plan, writes nothing, deletes nothing, and does not even
|
|
1290
|
+
> create a missing destination:
|
|
1291
|
+
>
|
|
1292
|
+
> ```python
|
|
1293
|
+
> preview = client.sync_collection(cid, dest, prune=True, dry_run=True)
|
|
1294
|
+
> print(f"{preview.plan.prune_candidates} local files would be deleted")
|
|
1295
|
+
> print(f"{preview.plan.to_download} would be fetched ({preview.plan.bytes:,} bytes)")
|
|
1296
|
+
> ```
|
|
1297
|
+
|
|
1298
|
+
**A partial sync does not raise.** Files that could not be fetched are counted
|
|
1299
|
+
in `failed`. Locally modified files that the daemon preserved are counted in
|
|
1300
|
+
`conflicts`. Either makes `is_complete` false because the destination was not
|
|
1301
|
+
brought fully in line. Files that were never going to arrive are counted
|
|
1302
|
+
separately in `skipped_external` and `skipped_unverified`, and deliberately do
|
|
1303
|
+
not affect `is_complete`, since the plan declared them up front.
|
|
1304
|
+
`get_collection_manifest()` names them individually.
|
|
1305
|
+
|
|
1306
|
+
**A sync cannot be cancelled.** The daemon runs it detached from the request, so
|
|
1307
|
+
`sync_timeout_s` bounds how long you *wait* — not the sync, which keeps running
|
|
1308
|
+
and keeps writing. Closing the client also abandons only your wait if the shared
|
|
1309
|
+
daemon remains alive. Only one sync may be active per destination; a second
|
|
1310
|
+
raises `DaemonError`.
|
|
1311
|
+
|
|
1312
|
+
### Updating an existing asset
|
|
1313
|
+
|
|
1314
|
+
Metadata and content are updated by two different calls, both taking an
|
|
1315
|
+
`asset_id`:
|
|
1316
|
+
|
|
1317
|
+
```python
|
|
1318
|
+
client.set_metadata(asset_id, tags=["v2"]) # metadata
|
|
1319
|
+
client.update_content(asset_id, "v2.csv") # a new content version
|
|
1320
|
+
```
|
|
1321
|
+
|
|
1322
|
+
Earlier versions are kept. List them and fetch an older one by ID:
|
|
1323
|
+
|
|
1324
|
+
```python
|
|
1325
|
+
for v in client.list_content_versions(asset_id):
|
|
1326
|
+
print(v.content_id, v.status, v.size_bytes, v.created_at)
|
|
1327
|
+
|
|
1328
|
+
client.download(asset_id, "out/", content_id="<older content_id>")
|
|
1329
|
+
```
|
|
1330
|
+
|
|
1331
|
+
A version appears in that list as soon as its upload starts, so an in-flight
|
|
1332
|
+
one shows up with `status="uploading"` and `is_downloadable == False` — it has
|
|
1333
|
+
no bytes behind it yet. They are surfaced rather than hidden so a concurrent
|
|
1334
|
+
upload doesn't look like nothing is happening.
|
|
1335
|
+
|
|
1336
|
+
**Order:** in-flight versions come first, then the available ones newest-first.
|
|
1337
|
+
So `[0]` is not reliably the newest downloadable version while an upload is
|
|
1338
|
+
running — filter rather than index:
|
|
1339
|
+
|
|
1340
|
+
```python
|
|
1341
|
+
newest = next(v for v in client.list_content_versions(asset_id)
|
|
1342
|
+
if v.is_downloadable)
|
|
1343
|
+
```
|
|
1344
|
+
|
|
1345
|
+
> **`upload()` with a title that already exists updates the content but
|
|
1346
|
+
> silently drops the metadata.** The daemon upserts by
|
|
1347
|
+
> `(collection, title, owner)`; when that matches, the console returns the
|
|
1348
|
+
> existing asset *unchanged* and attaches a new content version. The bytes
|
|
1349
|
+
> land, but `description`, `alias`, `record_type`, `tags` and `metadata` from
|
|
1350
|
+
> that call are ignored with no error. Use `update_content()` to add a version
|
|
1351
|
+
> to a known asset, and `set_metadata()` to change fields — both take an
|
|
1352
|
+
> `asset_id`, so neither can hit the wrong asset.
|
|
1353
|
+
|
|
1354
|
+
### Bulk operations
|
|
1355
|
+
|
|
1356
|
+
`upload()` takes a sequence of paths to build **one asset from several files** —
|
|
1357
|
+
the daemon puts them in a single `AssetContent` and moves them under one
|
|
1358
|
+
transfer. Filenames must be unique within the asset, since the filename is the
|
|
1359
|
+
server-side object key:
|
|
1360
|
+
|
|
1361
|
+
```python
|
|
1362
|
+
client.upload(["run.csv", "run.json"], title="Run 7",
|
|
1363
|
+
owner_type="project", owner_id=project_id)
|
|
1364
|
+
```
|
|
1365
|
+
|
|
1366
|
+
`upload_many()` is the other axis: **one asset per item**, uploaded with bounded
|
|
1367
|
+
concurrency. The keyword arguments are batch defaults and every `UploadItem`
|
|
1368
|
+
field overrides the default of the same name, so assets that differ in only a
|
|
1369
|
+
field or two stay readable:
|
|
1370
|
+
|
|
1371
|
+
```python
|
|
1372
|
+
from dataerai import UploadItem
|
|
1373
|
+
|
|
1374
|
+
result = client.upload_many(
|
|
1375
|
+
[
|
|
1376
|
+
UploadItem("s1.csv", title="Sample 1", metadata={"well": "A1"}),
|
|
1377
|
+
UploadItem("s2.csv", title="Sample 2", metadata={"well": "B2"}),
|
|
1378
|
+
UploadItem(["s3.csv", "s3.json"], title="Sample 3"),
|
|
1379
|
+
],
|
|
1380
|
+
owner_type="project",
|
|
1381
|
+
owner_id=project_id,
|
|
1382
|
+
record_type="dataset", # applies to all three
|
|
1383
|
+
metadata={"run": "2026-07-28"}, # merged into each item's own metadata
|
|
1384
|
+
concurrency=4,
|
|
1385
|
+
)
|
|
1386
|
+
result.raise_for_failures()
|
|
1387
|
+
```
|
|
1388
|
+
|
|
1389
|
+
`metadata` is **shallow-merged** — batch keys first, then the item's, item wins
|
|
1390
|
+
on collision. Every other field (including `tags`) replaces the default
|
|
1391
|
+
outright, so a shared tag can be dropped for one item.
|
|
1392
|
+
|
|
1393
|
+
`set_metadata_many()` does the same for metadata-only edits:
|
|
1394
|
+
|
|
1395
|
+
```python
|
|
1396
|
+
from dataerai import MetadataUpdate
|
|
1397
|
+
|
|
1398
|
+
client.set_metadata_many([
|
|
1399
|
+
MetadataUpdate(a_id, tags=["qc-pass"], metadata={"well": "A1"}),
|
|
1400
|
+
MetadataUpdate(b_id, tags=["qc-fail"], metadata={"well": "B2"}),
|
|
1401
|
+
]).raise_for_failures()
|
|
1402
|
+
```
|
|
1403
|
+
|
|
1404
|
+
Note this is **client-side fan-out, not a protocol batch**: the daemon exposes
|
|
1405
|
+
only a single-asset `asset.metadata.set`, so it issues one request per asset.
|
|
1406
|
+
You get bounded concurrency, one call, and one failure report — not fewer
|
|
1407
|
+
round-trips.
|
|
1408
|
+
|
|
1409
|
+
> **Both bulk calls finish the batch instead of aborting on the first failure.**
|
|
1410
|
+
> Aborting halfway through a large batch leaves you worse off than finishing and
|
|
1411
|
+
> reporting, so failures land in `result.failed` (each with the item's input
|
|
1412
|
+
> `index`) and successes in `result.succeeded`. A caller that ignores `failed`
|
|
1413
|
+
> will read a partial batch as a complete one — check `result.ok` or call
|
|
1414
|
+
> `result.raise_for_failures()`, which raises `BulkOperationError`.
|
|
1415
|
+
|
|
1416
|
+
`result.results` is positionally aligned with the input list — `results[i]` is
|
|
1417
|
+
the outcome of item `i`, or `None` if it failed. That is how you map an asset
|
|
1418
|
+
back to the item that produced it, which matters precisely because each item
|
|
1419
|
+
carries its own metadata:
|
|
1420
|
+
|
|
1421
|
+
```python
|
|
1422
|
+
for item, uploaded in zip(items, result.results):
|
|
1423
|
+
if uploaded is not None:
|
|
1424
|
+
print(item.title, "->", uploaded.asset_id)
|
|
1425
|
+
```
|
|
1426
|
+
|
|
1427
|
+
`succeeded` and `failed` are convenience views over the same outcomes, both in
|
|
1428
|
+
input order rather than completion order.
|
|
1429
|
+
|
|
1430
|
+
**On `concurrency`:** raising it past the default of 4 does not buy throughput.
|
|
1431
|
+
The daemon runs at most 4 transfers at once and queues the rest
|
|
1432
|
+
(`defaultMaxConcurrent` in `cli/internal/transfer/manager.go`, with no
|
|
1433
|
+
configuration knob), so a higher value only parks more SDK worker threads on
|
|
1434
|
+
transfers the daemon has not started. Note also that `transfer_timeout_s` starts
|
|
1435
|
+
when the daemon *accepts* a transfer, not when it starts moving bytes — time
|
|
1436
|
+
spent queued counts against it, so a large batch with a lowered timeout can fail
|
|
1437
|
+
items purely from queueing.
|
|
1438
|
+
|
|
1439
|
+
### ML adapters
|
|
1440
|
+
|
|
1441
|
+
The `dataerai.ml` package reads Dataerai-hosted Zarr image stores directly from
|
|
1442
|
+
the short-lived `/zarr-access/` bundle. PyTorch users can keep using
|
|
1443
|
+
`ZarrAssetDataset` / `ZarrOriginalsDataset`; Keras users get matching batch
|
|
1444
|
+
datasets:
|
|
1445
|
+
|
|
1446
|
+
```python
|
|
1447
|
+
from dataerai.ml import access
|
|
1448
|
+
from dataerai.ml.keras_dataset import ZarrOriginalsSequence
|
|
1449
|
+
|
|
1450
|
+
creds = access.load_credentials()
|
|
1451
|
+
train = ZarrOriginalsSequence(
|
|
1452
|
+
creds,
|
|
1453
|
+
asset_id="dataset-asset-id",
|
|
1454
|
+
size=64,
|
|
1455
|
+
batch_size=32,
|
|
1456
|
+
shuffle=True,
|
|
1457
|
+
label_mode="categorical",
|
|
1458
|
+
)
|
|
1459
|
+
|
|
1460
|
+
model.fit(train, epochs=5)
|
|
1461
|
+
```
|
|
1462
|
+
|
|
1463
|
+
For HLS4ML conversion, keep inputs in Keras' channels-last layout:
|
|
1464
|
+
|
|
1465
|
+
```python
|
|
1466
|
+
from dataerai.ml.hls4ml import convert_keras_model
|
|
1467
|
+
|
|
1468
|
+
hls_model = convert_keras_model(
|
|
1469
|
+
model,
|
|
1470
|
+
output_dir="hls-project",
|
|
1471
|
+
project_name="dataerai_model",
|
|
1472
|
+
data=train, # writes input/output .npy testbench arrays
|
|
1473
|
+
testbench_batches=2,
|
|
1474
|
+
)
|
|
1475
|
+
```
|
|
1476
|
+
|
|
1477
|
+
### `create_relationship()` arguments
|
|
1478
|
+
|
|
1479
|
+
Authors a directed provenance edge `from_asset_id` → `to_asset_id`. You need
|
|
1480
|
+
write access to the source and read access to the target. `rel_type` is a
|
|
1481
|
+
free-form verb describing the source's role, e.g. `"analysis_of"` or
|
|
1482
|
+
`"acquired_with"`. Notebook integrations can pass the same value with the
|
|
1483
|
+
keyword-only alias `relationship_type`.
|
|
1484
|
+
|
|
1485
|
+
| Argument | Type | Description |
|
|
1486
|
+
|---|---|---|
|
|
1487
|
+
| `from_asset_id` | `str` | Source (dependent) asset — the edge starts here (required) |
|
|
1488
|
+
| `to_asset_id` | `str` | Target (origin) asset — the edge points here (required) |
|
|
1489
|
+
| `rel_type` | `str \| None` | Free-form relationship type, ≤255 chars (required unless `relationship_type` is provided) |
|
|
1490
|
+
| `relationship_type` | `str \| None` | Keyword-only alias for `rel_type` |
|
|
1491
|
+
| `analysis_mode` | `str \| None` | `non_destructive`, `altering`, `destructive`, `in_situ`, `ex_situ`, `invasive`, `non_invasive` |
|
|
1492
|
+
| `qualifier_note` | `str \| None` | Free-text note on the relationship |
|
|
1493
|
+
| `qualifier_time` | `str \| None` | ISO-8601 timestamp |
|
|
1494
|
+
| `qualifiers` | `dict \| None` | JSON-serializable extra qualifiers |
|
|
1495
|
+
|
|
1496
|
+
```python
|
|
1497
|
+
# Link a processed result back to the raw data it came from.
|
|
1498
|
+
client.create_relationship(analysis.asset_id, raw.asset_id, "analysis_of",
|
|
1499
|
+
analysis_mode="non_destructive")
|
|
1500
|
+
```
|
|
1501
|
+
|
|
1502
|
+
### `upload()` keyword arguments
|
|
1503
|
+
|
|
1504
|
+
| Argument | Type | Description |
|
|
1505
|
+
|---|---|---|
|
|
1506
|
+
| `title` | `str` | Asset title (required) |
|
|
1507
|
+
| `owner_type` | `str` | `"project"` or `"user"` (required). `"user"` is accepted for compatibility but never stores the asset in your personal home area — the server files it in your default project and returns it with `owner_type="project"`. |
|
|
1508
|
+
| `owner_id` | `str` | Owner entity ID (required) |
|
|
1509
|
+
| `description` | `str \| None` | Free-text description |
|
|
1510
|
+
| `alias` | `str \| None` | Short identifier |
|
|
1511
|
+
| `tags` | `list[str] \| None` | Tag list |
|
|
1512
|
+
| `metadata` | `dict \| None` | Arbitrary key-value metadata |
|
|
1513
|
+
| `collection_id` | `str \| None` | Collection to add the asset to |
|
|
1514
|
+
| `chunk_size_mb` | `int \| None` | Override default 64 MiB chunk size |
|
|
1515
|
+
| `on_progress` | `Callable[[ProgressEvent], None] \| None` | Progress callback |
|
|
1516
|
+
| `transfer_timeout_s` | `float` | Max seconds to wait for completion (default 3600) |
|
|
1517
|
+
|
|
1518
|
+
### Progress events
|
|
1519
|
+
|
|
1520
|
+
```python
|
|
1521
|
+
@dataclass
|
|
1522
|
+
class ProgressEvent:
|
|
1523
|
+
transfer_id: str
|
|
1524
|
+
bytes_done: int
|
|
1525
|
+
bytes_total: int
|
|
1526
|
+
chunk_index: int
|
|
1527
|
+
chunk_count: int
|
|
1528
|
+
rate_bps: float
|
|
1529
|
+
file_index: int
|
|
1530
|
+
file_name: str
|
|
1531
|
+
|
|
1532
|
+
@property
|
|
1533
|
+
def percent(self) -> float: ... # 0–100
|
|
1534
|
+
|
|
1535
|
+
@property
|
|
1536
|
+
def rate_mbps(self) -> float: ...
|
|
1537
|
+
```
|
|
1538
|
+
|
|
1539
|
+
### Daemon events
|
|
1540
|
+
|
|
1541
|
+
`on_progress=` reports on one call's own transfer. `client.on()` subscribes to
|
|
1542
|
+
the daemon itself, which is the only way to observe a collection sync, a
|
|
1543
|
+
non-fatal `transfer.error`, or a pause.
|
|
1544
|
+
|
|
1545
|
+
```python
|
|
1546
|
+
client = DataeraiClient()
|
|
1547
|
+
client.connect()
|
|
1548
|
+
|
|
1549
|
+
@client.on("transfer.error")
|
|
1550
|
+
def on_error(evt):
|
|
1551
|
+
# Not a failure: when evt.retrying is set the daemon retries by itself.
|
|
1552
|
+
print(f"{evt.transfer_id}: {evt.message} (retrying={evt.retrying})")
|
|
1553
|
+
|
|
1554
|
+
@client.on("collection.sync_complete")
|
|
1555
|
+
def on_sync(evt):
|
|
1556
|
+
if evt.skipped_unverified:
|
|
1557
|
+
print(f"{evt.skipped_unverified} files skipped — no checksum offered")
|
|
1558
|
+
```
|
|
1559
|
+
|
|
1560
|
+
The daemon **broadcasts to every connected client**, so handlers also see work
|
|
1561
|
+
started by the desktop app or the CLI, not just this process.
|
|
1562
|
+
|
|
1563
|
+
> **Handlers run on the reader thread and must return promptly.** The daemon
|
|
1564
|
+
> buffers 64 events per connection and drops the overflow rather than blocking
|
|
1565
|
+
> other clients — a slow handler loses events outright, and they are not
|
|
1566
|
+
> redelivered. Queue the work; don't do it in the handler.
|
|
1567
|
+
>
|
|
1568
|
+
> In particular, **do not call a blocking client method from a handler.** The
|
|
1569
|
+
> reader thread is what delivers responses, so such a call waits for itself and
|
|
1570
|
+
> stalls until its request timeout. Put the id on a queue and act on it from
|
|
1571
|
+
> your own thread.
|
|
1572
|
+
|
|
1573
|
+
An unknown event name raises `ValueError` at registration rather than never
|
|
1574
|
+
firing. `dataerai.EVENT_NAMES` holds the full set.
|
|
1575
|
+
|
|
1576
|
+
Subscribe to `"*"` to see everything — for logging, or to find out what the
|
|
1577
|
+
daemon actually emits during an operation. A wildcard handler gets the same
|
|
1578
|
+
parsed event object, so use `event_name()` when the name matters:
|
|
1579
|
+
|
|
1580
|
+
```python
|
|
1581
|
+
from dataerai import event_name
|
|
1582
|
+
|
|
1583
|
+
client.on("*", lambda e: print(event_name(e), e))
|
|
1584
|
+
```
|
|
1585
|
+
|
|
1586
|
+
Handlers registered for an event's own name run before wildcard handlers.
|
|
1587
|
+
|
|
1588
|
+
Exceptions raised inside a handler never reach the reader thread, but they are
|
|
1589
|
+
**logged** to the `dataerai.events` logger rather than discarded — a handler
|
|
1590
|
+
with the wrong signature raises on every event, and a silent version of that is
|
|
1591
|
+
indistinguishable from the daemon never sending one.
|
|
1592
|
+
|
|
1593
|
+
Each distinct failure is reported **once**, with its traceback. A broken handler
|
|
1594
|
+
fails on every matching event and `transfer.progress` arrives continuously, so
|
|
1595
|
+
reporting every occurrence would bury the first traceback under thousands of
|
|
1596
|
+
copies. Two different broken handlers still get one report each. Silence the
|
|
1597
|
+
lot with `logging.getLogger("dataerai.events").setLevel(logging.CRITICAL)`.
|
|
1598
|
+
|
|
1599
|
+
> **Watching an `upload()` or `download()` finish? Subscribe to
|
|
1600
|
+
> `asset.upload_complete` / `asset.download_complete`, not `transfer.complete`.**
|
|
1601
|
+
> For an asset transfer the daemon sends the `asset.*` event *instead of* the
|
|
1602
|
+
> generic one — never both — so `transfer.complete` stays silent for exactly the
|
|
1603
|
+
> transfers the SDK starts. The same substitution applies to `transfer.failed`.
|
|
1604
|
+
|
|
1605
|
+
| Event | Payload | Notes |
|
|
1606
|
+
|---|---|---|
|
|
1607
|
+
| `transfer.progress` | `ProgressEvent` | Also available per-call via `on_progress=` |
|
|
1608
|
+
| `transfer.file_complete` | `FileCompleteEvent` | One file of a multi-file transfer |
|
|
1609
|
+
| `transfer.complete` | `TransferCompleteEvent` | **Not sent for asset transfers** — see the note above |
|
|
1610
|
+
| `transfer.error` | `TransferErrorEvent` | **Non-fatal.** `retrying` means the daemon retries by itself |
|
|
1611
|
+
| `transfer.failed` | `TransferFailedEvent` | Terminal, and **not sent for asset transfers**. A cancel arrives here too — there is no `transfer.cancelled` event |
|
|
1612
|
+
| `transfer.paused` | `TransferPausedEvent` | The confirmation for a fire-and-forget pause |
|
|
1613
|
+
| `asset.upload_complete` | `UploadCompleteEvent` | |
|
|
1614
|
+
| `asset.download_complete` | `DownloadCompleteEvent` | |
|
|
1615
|
+
| `asset.upload_failed` | `UploadFailedEvent` | |
|
|
1616
|
+
| `asset.download_failed` | `DownloadFailedEvent` | |
|
|
1617
|
+
| `collection.sync_progress` | `CollectionSyncProgressEvent` | Keyed by `sync_id`, not `transfer_id` |
|
|
1618
|
+
| `collection.sync_complete` | `CollectionSyncCompleteEvent` | `skipped_unverified > 0` means files were **not** written |
|
|
1619
|
+
|
|
1620
|
+
Names are the daemon's own wire names, so the string here is the string in the
|
|
1621
|
+
protocol reference and in the daemon's logs. Porting from the Node SDK, which
|
|
1622
|
+
spells them `transfer:fileComplete`: drop the camelCase and use `.` for `:`.
|
|
1623
|
+
|
|
1624
|
+
### Error types
|
|
1625
|
+
|
|
1626
|
+
| Exception | When raised |
|
|
1627
|
+
|---|---|
|
|
1628
|
+
| `DaemonError(code, message)` | Daemon returned a coded error (see `code` attribute) |
|
|
1629
|
+
| `DaemonTimeoutError` | Request or transfer exceeded the configured timeout |
|
|
1630
|
+
| `ConnectionError` | Daemon disconnected unexpectedly |
|