nominal 1.98.0__tar.gz → 1.100.0__tar.gz

{nominal-1.98.0 → nominal-1.100.0}/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [1.100.0](https://github.com/nominal-io/nominal-client/compare/v1.99.0...v1.100.0) (2025-12-19)
+
+
+### Features
+
+* add args and timestamp types for containerized ingest ([#551](https://github.com/nominal-io/nominal-client/issues/551)) ([1dd259a](https://github.com/nominal-io/nominal-client/commit/1dd259a299d83c55b482f443542f7feb63896795))
+* add clone/copy_from methods for assets, datasets, templates ([#548](https://github.com/nominal-io/nominal-client/issues/548)) ([f04c468](https://github.com/nominal-io/nominal-client/commit/f04c46800569da0ad6fb9440a336a7d8cac3542a))
+* added prefix_tree_delimiter parameter to get_or_create_dataset method ([#550](https://github.com/nominal-io/nominal-client/issues/550)) ([c204d94](https://github.com/nominal-io/nominal-client/commit/c204d9450d33a9742c0ae6b6a2d78e8baa796d2c))
+* allow listing runs on asset, deprecate search run by asset in client ([#541](https://github.com/nominal-io/nominal-client/issues/541)) ([35464e5](https://github.com/nominal-io/nominal-client/commit/35464e56ebc81577094b4ce0862d830e7d7bc92e))
+* allow promoting assets ([#542](https://github.com/nominal-io/nominal-client/issues/542)) ([1ce1082](https://github.com/nominal-io/nominal-client/commit/1ce1082dd025371bed4920c0e10d5ec93ac37687))
+* allow using truststore for ssl bypass ([#472](https://github.com/nominal-io/nominal-client/issues/472)) ([55a43c2](https://github.com/nominal-io/nominal-client/commit/55a43c23cb5a32b6cf90b2d50c5f0e79eeafaca7))
+* create new clone workbook and associated helpers in experimental ([#546](https://github.com/nominal-io/nominal-client/issues/546)) ([aeffb44](https://github.com/nominal-io/nominal-client/commit/aeffb44827b46e90101404d4a9a18d036aac49ab))
+* expose truststore.SSLContext across requests usage to permit usage in corporate networks ([55a43c2](https://github.com/nominal-io/nominal-client/commit/55a43c23cb5a32b6cf90b2d50c5f0e79eeafaca7))
+
+
+### Bug Fixes
+
+* fix log message missing argument ([#547](https://github.com/nominal-io/nominal-client/issues/547)) ([c684caa](https://github.com/nominal-io/nominal-client/commit/c684caa559544f7c5c2f13e6d94fd8f2e718827c))
+
+## [1.99.0](https://github.com/nominal-io/nominal-client/compare/v1.98.0...v1.99.0) (2025-12-04)
+
+
+### Features
+
+* allow ingesting .avro files ([#544](https://github.com/nominal-io/nominal-client/issues/544)) ([f5c4561](https://github.com/nominal-io/nominal-client/commit/f5c4561e1db6174a56d6b32b388ed7ad94679fdf))
+
 ## [1.98.0](https://github.com/nominal-io/nominal-client/compare/v1.97.0...v1.98.0) (2025-12-04)
 
 

{nominal-1.98.0 → nominal-1.100.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nominal
-Version: 1.98.0
+Version: 1.100.0
 Summary: Automate Nominal workflows in Python
 Project-URL: Homepage, https://nominal.io
 Project-URL: Documentation, https://docs.nominal.io
@@ -20,7 +20,7 @@ Requires-Dist: cachetools>=6.1.0
 Requires-Dist: click<9,>=8
 Requires-Dist: conjure-python-client<4,>=3.1.0
 Requires-Dist: ffmpeg-python>=0.2.0
-Requires-Dist: nominal-api==0.1019.0
+Requires-Dist: nominal-api==0.1032.0
 Requires-Dist: nominal-streaming==0.5.8
 Requires-Dist: openpyxl>=0.0.0
 Requires-Dist: pandas>=0.0.0
@@ -30,6 +30,7 @@ Requires-Dist: pyyaml>=0.0.0
 Requires-Dist: requests>=0.0.0
 Requires-Dist: rich>=14.1.0
 Requires-Dist: tabulate<0.10,>=0.9.0
+Requires-Dist: truststore>=0.10.4
 Requires-Dist: types-cachetools>=6.0.0.20250525
 Requires-Dist: typing-extensions<5,>=4
 Provides-Extra: protos

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/_utils/multipart.py

@@ -11,6 +11,7 @@ from typing import BinaryIO, Iterable
 import requests
 from nominal_api import ingest_api, upload_api
 
+from nominal.core._utils.networking import create_multipart_request_session
 from nominal.core.exceptions import NominalMultipartUploadFailed
 from nominal.core.filetype import FileType
 
@@ -22,6 +23,7 @@ DEFAULT_NUM_WORKERS = 8
 
 def _sign_and_upload_part_job(
     upload_client: upload_api.UploadService,
+    multipart_session: requests.Session,
     auth_header: str,
     key: str,
     upload_id: str,
@@ -45,8 +47,8 @@ def _sign_and_upload_part_job(
                     extra={"response.url": sign_response.url, **log_extras},
                 )
 
-                logger.debug("Pushing part %d for multipart upload", extra=log_extras)
-                put_response = requests.put(
+                logger.debug("Pushing part %d for multipart upload", part, extra=log_extras)
+                put_response = multipart_session.put(
                     sign_response.url,
                     data=data,
                     headers=sign_response.headers,
@@ -141,7 +143,10 @@ def put_multipart_upload(
     )
     initiate_response = upload_client.initiate_multipart_upload(auth_header, initiate_request)
     key, upload_id = initiate_response.key, initiate_response.upload_id
-    _sign_and_upload_part = partial(_sign_and_upload_part_job, upload_client, auth_header, key, upload_id, q)
+    multipart_session = create_multipart_request_session(pool_size=max_workers)
+    _sign_and_upload_part = partial(
+        _sign_and_upload_part_job, upload_client, multipart_session, auth_header, key, upload_id, q
+    )
 
     jobs: list[concurrent.futures.Future[requests.Response]] = []
 

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/_utils/multipart_downloader.py

@@ -13,11 +13,10 @@ from types import TracebackType
 from typing import Callable, Iterable, Mapping, Sequence, Type
 
 import requests
-from requests.adapters import HTTPAdapter
 from typing_extensions import Self
-from urllib3.util.retry import Retry
 
 from nominal.core._utils.multipart import DEFAULT_CHUNK_SIZE
+from nominal.core._utils.networking import create_multipart_request_session
 
 logger = logging.getLogger(__name__)
 
@@ -129,25 +128,12 @@ class MultipartFileDownloader:
             max_workers = multiprocessing.cpu_count()
             logger.info("Inferring core count as %d", max_workers)
 
-        session = cls._make_session(max_workers)
+        session = create_multipart_request_session(pool_size=max_workers)
         pool = ThreadPoolExecutor(max_workers=max_workers)
         return cls(max_workers, timeout, max_part_retries, _session=session, _pool=pool, _closed=False)
 
     # ---- lifecycle ----
 
-    @staticmethod
-    def _make_session(pool_size: int) -> requests.Session:
-        retries = Retry(
-            total=5,
-            backoff_factor=0.5,
-            status_forcelist=(429, 500, 502, 503, 504),
-            allowed_methods=frozenset(["GET", "HEAD"]),
-        )
-        s = requests.Session()
-        adapter = HTTPAdapter(max_retries=retries, pool_maxsize=pool_size)
-        s.mount("https://", adapter)
-        return s
-
     def close(self) -> None:
         if not self._closed:
             try:

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/_utils/networking.py

@@ -1,20 +1,69 @@
 from __future__ import annotations
 
 import gzip
+import logging
 import os
+import ssl
 from typing import Any, Callable, Mapping, Type, TypeVar
 
 import requests
+import truststore
 from conjure_python_client import ServiceConfiguration
-from conjure_python_client._http.requests_client import RetryWithJitter, TransportAdapter
-from requests.adapters import CaseInsensitiveDict
+from conjure_python_client._http.requests_client import KEEP_ALIVE_SOCKET_OPTIONS, RetryWithJitter
+from requests.adapters import DEFAULT_POOLSIZE, CaseInsensitiveDict, HTTPAdapter
+from urllib3.connection import HTTPConnection
+from urllib3.util.retry import Retry
+
+logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
 
 GZIP_COMPRESSION_LEVEL = 1
 
 
-class GzipRequestsAdapter(TransportAdapter):
+class SslBypassRequestsAdapter(HTTPAdapter):
+    """Transport adapter that allows customizing SSL options and forwarding host truststore.
+
+    NOTE: based on a combination of injecting `truststore.SSLContext` into
+        `conjure_python_client._http.requests_client.TransportAdapter`.
+    """
+
+    ENABLE_KEEP_ALIVE_ATTR = "_enable_keep_alive"
+    __attrs__ = [*HTTPAdapter.__attrs__, ENABLE_KEEP_ALIVE_ATTR]
+
+    def __init__(self, *args: Any, enable_keep_alive: bool = False, **kwargs: Any):
+        self._enable_keep_alive = enable_keep_alive
+        super().__init__(*args, **kwargs)
+
+    def init_poolmanager(
+        self,
+        connections: int,
+        maxsize: int,
+        block: bool = False,
+        **pool_kwargs: Mapping[str, Any],
+    ) -> None:
+        """Wrapper around the standard init_poolmanager from HTTPAdapter with modifications
+        to support keep-alive settings and injecting SSL context.
+        """
+        if self._enable_keep_alive:
+            keep_alive_kwargs: dict[str, Any] = {
+                "socket_options": [
+                    *HTTPConnection.default_socket_options,
+                    *KEEP_ALIVE_SOCKET_OPTIONS,
+                ]
+            }
+            pool_kwargs = {**pool_kwargs, **keep_alive_kwargs}
+
+        pool_kwargs["ssl_context"] = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+
+        super().init_poolmanager(connections, maxsize, block, **pool_kwargs)  # type: ignore[no-untyped-call]
+
+    def __setstate__(self, state: dict[str, Any]) -> None:
+        state[self.ENABLE_KEEP_ALIVE_ATTR] = state.get(self.ENABLE_KEEP_ALIVE_ATTR, False)
+        super().__setstate__(state)  # type: ignore[misc]
+
+
+class NominalRequestsAdapter(SslBypassRequestsAdapter):
     """Adapter used with `requests` library for sending gzip-compressed data.
 
     Based on: https://github.com/psf/requests/issues/1753#issuecomment-417806737
@@ -69,7 +118,7 @@ class GzipRequestsAdapter(TransportAdapter):
         return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
 
 
-def create_gzip_service_client(
+def create_conjure_service_client(
     service_class: Type[T],
     user_agent: str,
     service_config: ServiceConfiguration,
@@ -104,7 +153,7 @@ def create_gzip_service_client(
         status_forcelist=[308, 429, 503],
         backoff_factor=float(service_config.backoff_slot_size) / 1000,
     )
-    transport_adapter = GzipRequestsAdapter(max_retries=retry)
+    transport_adapter = NominalRequestsAdapter(max_retries=retry)
     # create a session, for shared connection polling, user agent, etc
     session = requests.Session()
     session.headers = CaseInsensitiveDict({"User-Agent": user_agent})
@@ -131,11 +180,11 @@ def create_conjure_client_factory(
 ) -> Callable[[Type[T]], T]:
     """Create factory method for creating conjure clients given the respective conjure service type
 
-    See `create_gzip_service_client` for documentation on parameters.
+    See `create_conjure_service_client` for documentation on parameters.
     """
 
     def factory(service_class: Type[T]) -> T:
-        return create_gzip_service_client(
+        return create_conjure_service_client(
             service_class,
             user_agent=user_agent,
             service_config=service_config,
@@ -143,3 +192,19 @@ def create_conjure_client_factory(
         )
 
     return factory
+
+
+def create_multipart_request_session(
+    *,
+    pool_size: int = DEFAULT_POOLSIZE,
+    num_retries: int = 5,
+) -> requests.Session:
+    retries = Retry(
+        total=num_retries,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+    )
+    session = requests.Session()
+    adapter = SslBypassRequestsAdapter(max_retries=retries, pool_maxsize=pool_size)
+    session.mount("https://", adapter)
+    return session

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/asset.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import logging
 from dataclasses import dataclass, field
 from types import MappingProxyType
 from typing import Iterable, Literal, Mapping, Protocol, Sequence, TypeAlias, cast
 
 from nominal_api import (
+    scout,
     scout_asset_api,
     scout_assets,
     scout_run_api,
@@ -13,6 +15,7 @@ from typing_extensions import Self
 
 from nominal.core._clientsbunch import HasScoutParams
 from nominal.core._utils.api_tools import HasRid, Link, RefreshableMixin, create_links, rid_from_instance_or_string
+from nominal.core._utils.pagination_tools import search_runs_by_asset_paginated
 from nominal.core.attachment import Attachment, _iter_get_attachments
 from nominal.core.connection import Connection, _get_connections
 from nominal.core.dataset import Dataset, _create_dataset, _get_datasets
@@ -22,6 +25,8 @@ from nominal.ts import IntegralNanosecondsUTC, _SecondsNanos
 
 ScopeType: TypeAlias = Connection | Dataset | Video
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass(frozen=True)
 class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
@@ -43,6 +48,8 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
     ):
         @property
         def assets(self) -> scout_assets.AssetService: ...
+        @property
+        def run(self) -> scout.RunService: ...
 
     @property
     def nominal_url(self) -> str:
@@ -98,6 +105,22 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
             if scope.data_source.type.lower() == stype
         }
 
+    def promote(self) -> Self:
+        """Promote this asset to be a standard, searchable, and displayable asset.
+
+        This method is only useful for assets that were created implicitly from creating a run directly on a dataset.
+        Nothing will happen from calling this method (aside from a logged warning) if called on a non-staged
+        asset (e.g. an asset created by create_asset, or an asset that's already been promoted).
+        """
+        if self._get_latest_api().is_staged:
+            request = scout_asset_api.UpdateAssetRequest(is_staged=False)
+            updated_asset = self._clients.assets.update_asset(self._clients.auth_header, request, self.rid)
+            self._refresh_from_api(updated_asset)
+        else:
+            logger.warning("Not promoting asset %s-- already promoted!", self.rid)
+
+        return self
+
     def get_data_scope(self, data_scope_name: str) -> ScopeType:
         """Retrieve a datascope by data scope name, or raise ValueError if one is not found."""
         for scope, data in self.list_data_scopes():
@@ -257,6 +280,7 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
         description: str | None = None,
         labels: Sequence[str] = (),
         properties: Mapping[str, str] | None = None,
+        prefix_tree_delimiter: str | None = None,
     ) -> Dataset:
         """Retrieve a dataset by data scope name, or create a new one if it does not exist."""
         try:
@@ -272,6 +296,10 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
                 workspace_rid=self._clients.workspace_rid,
             )
             dataset = Dataset._from_conjure(self._clients, enriched_dataset)
+
+            if prefix_tree_delimiter is not None:
+                dataset.set_channel_prefix_tree(prefix_tree_delimiter)
+
             self.add_dataset(data_scope_name, dataset)
             return dataset
 
@@ -372,6 +400,17 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
     def list_attachments(self) -> Sequence[Attachment]:
         return list(self._iter_list_attachments())
 
+    def list_runs(self) -> Sequence[Run]:
+        """List all runs associated with this Asset."""
+        return [
+            Run._from_conjure(self._clients, run)
+            for run in search_runs_by_asset_paginated(
+                self._clients.run,
+                self._clients.auth_header,
+                self.rid,
+            )
+        ]
+
     def remove_attachments(self, attachments: Iterable[Attachment] | Iterable[str]) -> None:
         """Remove attachments from this asset.
         Does not remove the attachments from Nominal.
@@ -403,3 +442,7 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
             created_at=_SecondsNanos.from_flexible(asset.created_at).to_nanoseconds(),
             _clients=clients,
         )
+
+
+# Moving to bottom to deal with circular dependencies
+from nominal.core.run import Run  # noqa: E402

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/client.py

@@ -608,6 +608,10 @@ class NominalClient:
             )
         )
 
+    @deprecated(
+        "NominalClient.search_runs_by_asset is deprecated and will be removed in a future version. "
+        "Use Asset.list_runs() instead."
+    )
     def search_runs_by_asset(self, asset: Asset | str) -> Sequence[Run]:
         """Search for all runs associated with a given asset:
 

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/containerized_extractors.py

@@ -5,7 +5,7 @@ from enum import Enum
 from typing import Mapping, Protocol, Sequence
 
 from nominal_api import ingest_api
-from typing_extensions import Self
+from typing_extensions import Self, deprecated
 
 from nominal._utils.dataclass_tools import update_dataclass
 from nominal.core._clientsbunch import HasScoutParams
@@ -203,9 +203,22 @@ class ContainerizedExtractor(HasRid):
     inputs: Sequence[FileExtractionInput]
     properties: Mapping[str, str]
     labels: Sequence[str]
-    timestamp_metadata: TimestampMetadata
+    default_timestamp_metadata: TimestampMetadata | None
     _clients: _Clients = field(repr=False)
 
+    @property
+    @deprecated(
+        "The `timestamp_metadata` field of a ContainerizedExtractor is deprecated and will be removed in a future "
+        "release. Use the `default_timestamp_metadata` field instead."
+    )
+    def timestamp_metadata(self) -> TimestampMetadata:
+        if self.default_timestamp_metadata is None:
+            raise ValueError(
+                f"Containerized extractor {self.name} ({self.rid}) has no default configured timestamp metadata"
+            )
+
+        return self.default_timestamp_metadata
+
     class _Clients(HasScoutParams, Protocol):
         @property
         def containerized_extractors(self) -> ingest_api.ContainerizedExtractorService: ...
@@ -251,6 +264,12 @@ class ContainerizedExtractor(HasRid):
 
     @classmethod
     def _from_conjure(cls, clients: _Clients, raw_extractor: ingest_api.ContainerizedExtractor) -> Self:
+        timestamp_metadata = (
+            None
+            if raw_extractor.timestamp_metadata is None
+            else TimestampMetadata._from_conjure(raw_extractor.timestamp_metadata)
+        )
+
         return cls(
             rid=raw_extractor.rid,
             name=raw_extractor.name,
@@ -259,6 +278,6 @@ class ContainerizedExtractor(HasRid):
             inputs=[FileExtractionInput._from_conjure(raw_input) for raw_input in raw_extractor.inputs],
             properties=raw_extractor.properties,
             labels=raw_extractor.labels,
-            timestamp_metadata=TimestampMetadata._from_conjure(raw_extractor.timestamp_metadata),
+            default_timestamp_metadata=timestamp_metadata,
             _clients=clients,
         )

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/dataset.py

@@ -6,7 +6,7 @@ from datetime import timedelta
 from io import TextIOBase
 from pathlib import Path
 from types import MappingProxyType
-from typing import BinaryIO, Iterable, Mapping, Sequence, TypeAlias
+from typing import BinaryIO, Iterable, Mapping, Sequence, TypeAlias, overload
 
 from nominal_api import api, ingest_api, scout_catalog
 from typing_extensions import Self, deprecated
@@ -203,6 +203,78 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
     # Backward compatibility
     add_to_dataset_from_io = add_from_io
 
+    def add_avro_stream(
+        self,
+        path: Path | str,
+    ) -> DatasetFile:
+        """Upload an avro stream file with a specific schema, described below.
+
+        This is a "stream-like" file format to support
+        use cases where a columnar/tabular format does not make sense. This closely matches Nominal's streaming
+        API, making it useful for use cases where network connection drops during streaming and a backup file needs
+        to be created.
+
+        If this schema is not used, will result in a failed ingestion.
+        {
+            "type": "record",
+            "name": "AvroStream",
+            "namespace": "io.nominal.ingest",
+            "fields": [
+                {
+                    "name": "channel",
+                    "type": "string",
+                    "doc": "Channel/series name (e.g., 'vehicle_id', 'col_1', 'temperature')",
+                },
+                {
+                    "name": "timestamps",
+                    "type": {"type": "array", "items": "long"},
+                    "doc": "Array of Unix timestamps in nanoseconds",
+                },
+                {
+                    "name": "values",
+                    "type": {"type": "array", "items": ["double", "string"]},
+                    "doc": "Array of values. Can either be doubles or strings",
+                },
+                {
+                    "name": "tags",
+                    "type": {"type": "map", "values": "string"},
+                    "default": {},
+                    "doc": "Key-value metadata tags",
+                },
+            ],
+        }
+
+        Args:
+            path: Path to the .avro file to upload
+
+        Returns:
+            Reference to the ingesting DatasetFile
+
+        """
+        avro_path = Path(path)
+        s3_path = upload_multipart_file(
+            self._clients.auth_header,
+            self._clients.workspace_rid,
+            avro_path,
+            self._clients.upload,
+            file_type=FileTypes.AVRO_STREAM,
+        )
+        target = ingest_api.DatasetIngestTarget(
+            existing=ingest_api.ExistingDatasetIngestDestination(dataset_rid=self.rid)
+        )
+        resp = self._clients.ingest.ingest(
+            self._clients.auth_header,
+            ingest_api.IngestRequest(
+                options=ingest_api.IngestOptions(
+                    avro_stream=ingest_api.AvroStreamOpts(
+                        source=ingest_api.IngestSource(s3=ingest_api.S3IngestSource(s3_path)),
+                        target=target,
+                    )
+                )
+            ),
+        )
+        return self._handle_ingest_response(resp)
+
     def add_journal_json(
         self,
         path: Path | str,
@@ -338,11 +410,38 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
     # Backward compatibility
     add_ardupilot_dataflash_to_dataset = add_ardupilot_dataflash
 
+    @overload
     def add_containerized(
         self,
         extractor: str | ContainerizedExtractor,
         sources: Mapping[str, Path | str],
         tag: str | None = None,
+        *,
+        arguments: Mapping[str, str] | None = None,
+        tags: Mapping[str, str] | None = None,
+    ) -> DatasetFile: ...
+    @overload
+    def add_containerized(
+        self,
+        extractor: str | ContainerizedExtractor,
+        sources: Mapping[str, Path | str],
+        tag: str | None = None,
+        *,
+        arguments: Mapping[str, str] | None = None,
+        tags: Mapping[str, str] | None = None,
+        timestamp_column: str,
+        timestamp_type: _AnyTimestampType,
+    ) -> DatasetFile: ...
+    def add_containerized(
+        self,
+        extractor: str | ContainerizedExtractor,
+        sources: Mapping[str, Path | str],
+        tag: str | None = None,
+        *,
+        arguments: Mapping[str, str] | None = None,
+        tags: Mapping[str, str] | None = None,
+        timestamp_column: str | None = None,
+        timestamp_type: _AnyTimestampType | None = None,
     ) -> DatasetFile:
         """Add data from proprietary data formats using a pre-registered custom extractor.
 
@@ -352,7 +451,24 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
                 NOTE: these must match the registered inputs of the containerized extractor exactly
             tag: Tag of the Docker container which hosts the extractor.
                 NOTE: if not provided, the default registered docker tag will be used.
+            arguments: Mapping of key-value pairs of input arguments to the extractor.
+            tags: Key-value pairs of tags to apply to all data ingested from the containerized extractor run.
+            timestamp_column: the column in the dataset that contains the timestamp data.
+                NOTE: this is applied uniformly to all output files
+                NOTE: must be provided with a `timestamp_type` or a ValueError will be raised
+            timestamp_type: the type of timestamp data in the dataset.
+                NOTE: this is applied uniformly to all output files
+                NOTE: must be provided with a `timestamp_column` or a ValueError will be raised
         """
+        timestamp_metadata = None
+        if timestamp_column is not None and timestamp_type is not None:
+            timestamp_metadata = ingest_api.TimestampMetadata(
+                series_name=timestamp_column,
+                timestamp_type=_to_typed_timestamp_type(timestamp_type)._to_conjure_ingest_api(),
+            )
+        elif None in (timestamp_column, timestamp_type):
+            raise ValueError("Only one of `timestamp_column` and `timestamp_type` provided!")
+
         if isinstance(extractor, str):
             extractor = ContainerizedExtractor._from_conjure(
                 self._clients,
@@ -379,13 +495,14 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
             )
             logger.info("Uploaded %s -> %s", source_path, s3_path)
             s3_inputs[source] = s3_path
+
         logger.info("Triggering custom extractor %s (tag=%s) with %s", extractor.name, tag, s3_inputs)
         resp = self._clients.ingest.ingest(
             self._clients.auth_header,
             trigger_ingest=ingest_api.IngestRequest(
                 options=ingest_api.IngestOptions(
                     containerized=ingest_api.ContainerizedOpts(
-                        arguments={},
+                        arguments={**(arguments or {})},
                         extractor_rid=extractor.rid,
                         sources={
                             source: ingest_api.IngestSource(s3=ingest_api.S3IngestSource(path=s3_path))
@@ -395,6 +512,8 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
                             existing=ingest_api.ExistingDatasetIngestDestination(self.rid)
                         ),
                         tag=tag,
+                        additional_file_tags={**(tags or {})},
+                        timestamp_metadata=timestamp_metadata,
                     )
                 )
             ),

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/filetype.py

@@ -111,6 +111,7 @@ class FileType(NamedTuple):
 
 
 class FileTypes:
+    AVRO_STREAM: FileType = FileType(".avro", "application/avro")
     BINARY: FileType = FileType("", "application/octet-stream")
     CSV: FileType = FileType(".csv", "text/csv")
     CSV_GZ: FileType = FileType(".csv.gz", "text/csv")

{nominal-1.98.0 → nominal-1.100.0}/nominal/core/run.py

@@ -6,11 +6,11 @@ from types import MappingProxyType
 from typing import Iterable, Mapping, Protocol, Sequence, cast
 
 from nominal_api import (
-    scout,
     scout_run_api,
 )
 from typing_extensions import Self
 
+from nominal.core import asset as core_asset
 from nominal.core._clientsbunch import HasScoutParams
 from nominal.core._utils.api_tools import (
     HasRid,
@@ -20,7 +20,6 @@ from nominal.core._utils.api_tools import (
     create_links,
     rid_from_instance_or_string,
 )
-from nominal.core.asset import Asset
 from nominal.core.attachment import Attachment, _iter_get_attachments
 from nominal.core.connection import Connection, _get_connections
 from nominal.core.dataset import Dataset, _get_datasets
@@ -45,12 +44,11 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run]):
     _clients: _Clients = field(repr=False)
 
     class _Clients(
-        Asset._Clients,
+        core_asset.Asset._Clients,
         HasScoutParams,
         Protocol,
     ):
-        @property
-        def run(self) -> scout.RunService: ...
+        pass
 
     @property
     def nominal_url(self) -> str:
@@ -304,13 +302,13 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run]):
         """List a sequence of Attachments associated with this Run."""
         return list(self._iter_list_attachments())
 
-    def _iter_list_assets(self) -> Iterable[Asset]:
+    def _iter_list_assets(self) -> Iterable[core_asset.Asset]:
         run = self._get_latest_api()
         assets = self._clients.assets.get_assets(self._clients.auth_header, run.assets)
         for a in assets.values():
-            yield Asset._from_conjure(self._clients, a)
+            yield core_asset.Asset._from_conjure(self._clients, a)
 
-    def list_assets(self) -> Sequence[Asset]:
+    def list_assets(self) -> Sequence[core_asset.Asset]:
         """List assets associated with this run."""
         return list(self._iter_list_assets())