nominal 1.109.0__py3-none-any.whl → 1.111.0__py3-none-any.whl

nominal/core/dataset.py

@@ -26,6 +26,8 @@ from nominal.core.filetype import FileType, FileTypes
 from nominal.core.log import LogPoint, _write_logs
 from nominal.ts import (
     _AnyTimestampType,
+    _InferrableTimestampType,
+    _SecondsNanos,
     _to_typed_timestamp_type,
 )
 
@@ -93,6 +95,32 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
         updated_dataset = self._clients.catalog.update_dataset_metadata(self._clients.auth_header, self.rid, request)
         return self._refresh_from_api(updated_dataset)
 
+    def update_bounds(
+        self,
+        *,
+        start: _InferrableTimestampType,
+        end: _InferrableTimestampType,
+    ) -> Self:
+        """Update the bounds (start and end timestamps) of the dataset.
+        Updates the current instance, and returns it.
+
+        Args:
+            start: The start timestamp of the dataset bounds. Can be a datetime, ISO 8601 string,
+                or integer nanoseconds since epoch.
+            end: The end timestamp of the dataset bounds. Can be a datetime, ISO 8601 string,
+                or integer nanoseconds since epoch.
+
+        Returns:
+            The updated Dataset instance with new bounds.
+        """
+        bounds = Bounds(
+            start=_SecondsNanos.from_flexible(start).to_nanoseconds(),
+            end=_SecondsNanos.from_flexible(end).to_nanoseconds(),
+        )
+        request = scout_catalog.UpdateBoundsRequest(bounds=bounds._to_conjure())
+        updated_dataset = self._clients.catalog.update_bounds(self._clients.auth_header, request, self.rid)
+        return self._refresh_from_api(updated_dataset)
+
     def _handle_ingest_response(self, response: ingest_api.IngestResponse) -> DatasetFile:
         if response.details.dataset is None:
             raise ValueError(f"Expected response to provide dataset details, received: {response.details.type}")
@@ -216,6 +244,8 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
         API, making it useful for use cases where network connection drops during streaming and a backup file needs
         to be created.
 
+        For struct columns, values should be converted to JSON strings and wrapped in the JsonStruct record type.
+
         If this schema is not used, will result in a failed ingestion.
         {
             "type": "record",
@@ -234,8 +264,15 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
                 },
                 {
                     "name": "values",
-                    "type": {"type": "array", "items": ["double", "string"]},
-                    "doc": "Array of values. Can either be doubles or strings",
+                    "type": {"type": "array", "items": [
+                        "double",
+                        "string",
+                        "long",
+                        {"type": "record", "name": "DoubleArray", "fields": [{"name": "items", "type": {"type": "array", "items": "double"}}]},
+                        {"type": "record", "name": "StringArray", "fields": [{"name": "items", "type": {"type": "array", "items": "string"}}]},
+                        {"type": "record", "name": "JsonStruct", "fields": [{"name": "json", "type": "string"}]}
+                    ]},
+                    "doc": "Array of values. Can be doubles, longs, strings, arrays, or JSON structs",
                 },
                 {
                     "name": "tags",
@@ -246,6 +283,8 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
             ],
         }
 
+        Note: The previous schema with only "double" and "string" value types is still fully supported.
+
         Args:
             path: Path to the .avro file to upload
 

nominal/core/dataset_file.py

@@ -280,6 +280,8 @@ class IngestStatus(Enum):
     SUCCESS = "SUCCESS"
     IN_PROGRESS = "IN_PROGRESS"
     FAILED = "FAILED"
+    DELETION_IN_PROGRESS = "DELETION_IN_PROGRESS"
+    DELETED = "DELETED"
 
     @classmethod
     def _from_conjure(cls, status: api.IngestStatusV2) -> IngestStatus:
@@ -289,6 +291,10 @@ class IngestStatus(Enum):
             return cls.IN_PROGRESS
         elif status.error is not None:
             return cls.FAILED
+        elif status.deletion_in_progress is not None:
+            return cls.DELETION_IN_PROGRESS
+        elif status.deleted is not None:
+            return cls.DELETED
         raise ValueError(f"Unknown ingest status: {status.type}")
 
 

nominal/core/datasource.py

@@ -18,7 +18,6 @@ from nominal_api import (
     storage_writer_api,
     timeseries_channelmetadata,
     timeseries_channelmetadata_api,
-    timeseries_logicalseries,
     timeseries_metadata,
     timeseries_metadata_api,
     upload_api,
@@ -53,8 +52,6 @@ class DataSource(HasRid):
         @property
         def datasource(self) -> scout_datasource.DataSourceService: ...
         @property
-        def logical_series(self) -> timeseries_logicalseries.LogicalSeriesService: ...
-        @property
         def units(self) -> scout.UnitsService: ...
         @property
         def ingest(self) -> ingest_api.IngestService: ...

nominal/core/run.py

@@ -3,16 +3,17 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from types import MappingProxyType
-from typing import Iterable, Mapping, Protocol, Sequence, cast
+from typing import TYPE_CHECKING, Iterable, Mapping, Protocol, Sequence, cast
 
 from nominal_api import (
+    event,
+    scout,
     scout_asset_api,
+    scout_assets,
     scout_run_api,
 )
 from typing_extensions import Self
 
-from nominal.core import asset as core_asset
-from nominal.core._clientsbunch import HasScoutParams
 from nominal.core._event_types import EventType
 from nominal.core._utils.api_tools import (
     HasRid,
@@ -20,16 +21,20 @@ from nominal.core._utils.api_tools import (
     LinkDict,
     RefreshableMixin,
     create_links,
+    filter_scopes,
     rid_from_instance_or_string,
 )
-from nominal.core.asset import _filter_scopes
 from nominal.core.attachment import Attachment, _iter_get_attachments
 from nominal.core.connection import Connection, _get_connections
 from nominal.core.dataset import Dataset, _DatasetWrapper, _get_datasets
+from nominal.core.datasource import DataSource
 from nominal.core.event import Event, _create_event
 from nominal.core.video import Video, _get_video
 from nominal.ts import IntegralNanosecondsDuration, IntegralNanosecondsUTC, _SecondsNanos, _to_api_duration
 
+if TYPE_CHECKING:
+    from nominal.core.asset import Asset
+
 
 @dataclass(frozen=True)
 class Run(HasRid, RefreshableMixin[scout_run_api.Run], _DatasetWrapper):
@@ -48,11 +53,17 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run], _DatasetWrapper):
     _clients: _Clients = field(repr=False)
 
     class _Clients(
-        core_asset.Asset._Clients,
-        HasScoutParams,
+        Attachment._Clients,
+        DataSource._Clients,
+        Video._Clients,
         Protocol,
     ):
-        pass
+        @property
+        def assets(self) -> scout_assets.AssetService: ...
+        @property
+        def event(self) -> event.EventService: ...
+        @property
+        def run(self) -> scout.RunService: ...
 
     @property
     def nominal_url(self) -> str:
@@ -105,7 +116,7 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run], _DatasetWrapper):
         if len(api_run.assets) > 1:
             raise RuntimeError("Can't retrieve dataset scopes on multi-asset runs")
 
-        return _filter_scopes(api_run.asset_data_scopes, "dataset")
+        return filter_scopes(api_run.asset_data_scopes, "dataset")
 
     def _list_datasource_rids(
         self, datasource_type: str | None = None, property_name: str | None = None
@@ -350,13 +361,16 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run], _DatasetWrapper):
         """List a sequence of Attachments associated with this Run."""
         return list(self._iter_list_attachments())
 
-    def _iter_list_assets(self) -> Iterable[core_asset.Asset]:
+    def _iter_list_assets(self) -> Iterable["Asset"]:
+        from nominal.core.asset import Asset
+
+        clients = cast(Asset._Clients, self._clients)
         run = self._get_latest_api()
         assets = self._clients.assets.get_assets(self._clients.auth_header, run.assets)
         for a in assets.values():
-            yield core_asset.Asset._from_conjure(self._clients, a)
+            yield Asset._from_conjure(clients, a)
 
-    def list_assets(self) -> Sequence[core_asset.Asset]:
+    def list_assets(self) -> Sequence["Asset"]:
         """List assets associated with this run."""
         return list(self._iter_list_assets())
 

nominal/core/streaming_checklist.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Iterable, Protocol
+
+from nominal_api import scout_checklistexecution_api
+
+from nominal.core._clientsbunch import HasScoutParams
+from nominal.core._utils.pagination_tools import (
+    list_streaming_checklists_for_asset_paginated,
+    list_streaming_checklists_paginated,
+)
+
+
+class _Clients(HasScoutParams, Protocol):
+    @property
+    def checklist_execution(self) -> scout_checklistexecution_api.ChecklistExecutionService: ...
+
+
+def _iter_list_streaming_checklists(
+    clients: _Clients,
+    asset_rid: str | None = None,
+) -> Iterable[str]:
+    if asset_rid is None:
+        return list_streaming_checklists_paginated(clients.checklist_execution, clients.auth_header)
+    return list_streaming_checklists_for_asset_paginated(clients.checklist_execution, clients.auth_header, asset_rid)

nominal/core/video.py

@@ -8,9 +8,9 @@ from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from io import BytesIO, TextIOBase, TextIOWrapper
 from types import MappingProxyType
-from typing import BinaryIO, Mapping, Protocol, Sequence
+from typing import BinaryIO, Mapping, Protocol, Sequence, overload
 
-from nominal_api import api, ingest_api, scout_video, scout_video_api, upload_api
+from nominal_api import api, ingest_api, scout_catalog, scout_video, scout_video_api, upload_api
 from typing_extensions import Self
 
 from nominal.core._clientsbunch import HasScoutParams
@@ -44,6 +44,8 @@ class Video(HasRid, RefreshableMixin[scout_video_api.Video]):
         def ingest(self) -> ingest_api.IngestService: ...
         @property
         def video_file(self) -> scout_video.VideoFileService: ...
+        @property
+        def catalog(self) -> scout_catalog.CatalogService: ...
 
     def poll_until_ingestion_completed(self, interval: timedelta = timedelta(seconds=1)) -> None:
         """Block until video ingestion has completed.
@@ -117,18 +119,38 @@ class Video(HasRid, RefreshableMixin[scout_video_api.Video]):
         """Unarchives this video, allowing it to show up in the 'All Videos' pane in the UI."""
         self._clients.video.unarchive(self._clients.auth_header, self.rid)
 
+    @overload
+    def add_file(
+        self,
+        path: PathLike,
+        *,
+        start: datetime | IntegralNanosecondsUTC,
+        description: str | None = None,
+    ) -> VideoFile: ...
+
+    @overload
     def add_file(
         self,
         path: PathLike,
+        *,
+        frame_timestamps: Sequence[IntegralNanosecondsUTC],
+        description: str | None = None,
+    ) -> VideoFile: ...
+
+    def add_file(
+        self,
+        path: PathLike,
+        *,
         start: datetime | IntegralNanosecondsUTC | None = None,
         frame_timestamps: Sequence[IntegralNanosecondsUTC] | None = None,
         description: str | None = None,
     ) -> VideoFile:
-        """Append to a video from a file-path to H264-encoded video data.
+        """Append to a video from a file-path to H264-encoded video data. Only one of start or frame_timestamps
+        is allowed.
 
         Args:
             path: Path to the video file to add to an existing video within Nominal
-            start: Starting timestamp of the video file in absolute UTC time
+            start: Starting timestamp of the video file in absolute UTC time.
             frame_timestamps: Per-frame absolute nanosecond timestamps. Most usecases should instead use the 'start'
                 parameter, unless precise per-frame metadata is available and desired.
             description: Description of the video file.
@@ -141,16 +163,46 @@ class Video(HasRid, RefreshableMixin[scout_video_api.Video]):
         file_type = FileType.from_video(path)
 
         with path.open("rb") as video_file:
-            return self.add_from_io(
-                video_file,
-                name=path_upload_name(path, file_type),
-                start=start,
-                frame_timestamps=frame_timestamps,
-                description=description,
-                file_type=file_type,
-            )
+            if start is not None:
+                return self.add_from_io(
+                    video_file,
+                    name=path_upload_name(path, file_type),
+                    start=start,
+                    description=description,
+                    file_type=file_type,
+                )
+            elif frame_timestamps is not None:
+                return self.add_from_io(
+                    video_file,
+                    name=path_upload_name(path, file_type),
+                    frame_timestamps=frame_timestamps,
+                    description=description,
+                    file_type=file_type,
+                )
+            else:  # This should never be reached due to the validation above
+                raise ValueError("Either 'start' or 'frame_timestamps' must be provided")
 
-    add_file_to_video = add_file
+    @overload
+    def add_from_io(
+        self,
+        video: BinaryIO,
+        name: str,
+        *,
+        start: datetime | IntegralNanosecondsUTC,
+        description: str | None = None,
+        file_type: tuple[str, str] | FileType = FileTypes.MP4,
+    ) -> VideoFile: ...
+
+    @overload
+    def add_from_io(
+        self,
+        video: BinaryIO,
+        name: str,
+        *,
+        frame_timestamps: Sequence[IntegralNanosecondsUTC],
+        description: str | None = None,
+        file_type: tuple[str, str] | FileType = FileTypes.MP4,
+    ) -> VideoFile: ...
 
     def add_from_io(
         self,
@@ -179,6 +231,12 @@ class Video(HasRid, RefreshableMixin[scout_video_api.Video]):
         if isinstance(video, TextIOBase):
             raise TypeError(f"video {video} must be open in binary mode, rather than text mode")
 
+        # Validation: ensure exactly one of start or frame_timestamps is provided
+        if start is None and frame_timestamps is None:
+            raise ValueError("Either 'start' or 'frame_timestamps' must be provided")
+        if start is not None and frame_timestamps is not None:
+            raise ValueError("Only one of 'start' or 'frame_timestamps' may be provided")
+
         timestamp_manifest = _build_video_file_timestamp_manifest(
             self._clients.auth_header, self._clients.workspace_rid, self._clients.upload, start, frame_timestamps
         )

nominal/core/video_file.py

@@ -4,13 +4,14 @@ import logging
 import time
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
-from typing import Protocol
+from typing import Protocol, Tuple
 
-from nominal_api import scout_video, scout_video_api
+from nominal_api import scout_catalog, scout_video, scout_video_api
 from typing_extensions import Self
 
 from nominal.core._clientsbunch import HasScoutParams
 from nominal.core._utils.api_tools import HasRid, RefreshableMixin
+from nominal.core._video_types import McapVideoDetails, TimestampOptions
 from nominal.core.exceptions import NominalIngestError, NominalIngestFailed
 from nominal.ts import IntegralNanosecondsUTC, _SecondsNanos
 
@@ -28,6 +29,8 @@ class VideoFile(HasRid, RefreshableMixin[scout_video_api.VideoFile]):
     class _Clients(HasScoutParams, Protocol):
         @property
         def video_file(self) -> scout_video.VideoFileService: ...
+        @property
+        def catalog(self) -> scout_catalog.CatalogService: ...
 
     def archive(self) -> None:
         """Archive the video file, disallowing it to appear when playing back the video"""
@@ -128,6 +131,63 @@ class VideoFile(HasRid, RefreshableMixin[scout_video_api.VideoFile]):
 
             time.sleep(interval.total_seconds())
 
+    def _get_file_ingest_options(self) -> Tuple[McapVideoDetails | None, TimestampOptions | None]:
+        """Get ingest options metadata for this video file.
+
+        Retrieves metadata about the video file (such as timestamps, frame rate, and scale factor)
+        that can be used when ingesting this video into a video channel. The returned options
+        are either MCAP or MISC metadata depending on the video file type.
+
+        Returns:
+            Video file ingest options (either McapVideoFileMetadata or MiscVideoFileMetadata).
+
+        Raises:
+            ValueError: If the video file has an unexpected timestamp manifest type.
+        """
+        api_video_file = self._get_latest_api()
+        if api_video_file.origin_metadata.timestamp_manifest.type == "mcap":
+            mcap_manifest = api_video_file.origin_metadata.timestamp_manifest.mcap
+            topic = (
+                mcap_manifest.mcap_channel_locator.topic
+                if mcap_manifest and mcap_manifest.mcap_channel_locator and mcap_manifest.mcap_channel_locator.topic
+                else ""
+            )
+            mcap_video_details = McapVideoDetails(
+                mcap_channel_locator_topic=topic,
+            )
+            return (mcap_video_details, None)
+        else:
+            # TODO(sean): We need to add support for if starting timestamp isn't present, aka we have frame timestamps
+            # from S3.
+            if api_video_file.origin_metadata.timestamp_manifest.no_manifest is None:
+                raise NotImplementedError(
+                    f"Expected no_manifest timestamp manifest for non-MCAP video file, "
+                    f"but got type: {api_video_file._origin_metadata._timestamp_manifest._type}"
+                )
+            if api_video_file.segment_metadata is None:
+                raise ValueError(
+                    "Expected segment metadata for non-MCAP video file: %s", api_video_file.segment_metadata
+                )
+            if (
+                api_video_file.segment_metadata.max_absolute_timestamp is None
+                or api_video_file.segment_metadata.scale_factor is None
+                or api_video_file.segment_metadata.media_frame_rate is None
+            ):
+                raise ValueError(
+                    "Not all timestamp metadata is populated in segment metadata: %s", api_video_file.segment_metadata
+                )
+            video_file_ingest_options = TimestampOptions(
+                starting_timestamp=_SecondsNanos.from_api(
+                    api_video_file.origin_metadata.timestamp_manifest.no_manifest.starting_timestamp
+                ).to_nanoseconds(),
+                ending_timestamp=_SecondsNanos.from_api(
+                    api_video_file.segment_metadata.max_absolute_timestamp
+                ).to_nanoseconds(),
+                scaling_factor=api_video_file.segment_metadata.scale_factor,
+                true_framerate=api_video_file.segment_metadata.media_frame_rate,
+            )
+            return (None, video_file_ingest_options)
+
     @classmethod
     def _from_conjure(cls, clients: _Clients, video_file: scout_video_api.VideoFile) -> Self:
         return cls(