nominal 1.97.0__tar.gz → 1.99.0__tar.gz

{nominal-1.97.0 → nominal-1.99.0}/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [1.99.0](https://github.com/nominal-io/nominal-client/compare/v1.98.0...v1.99.0) (2025-12-04)
+
+
+### Features
+
+* allow ingesting .avro files ([#544](https://github.com/nominal-io/nominal-client/issues/544)) ([f5c4561](https://github.com/nominal-io/nominal-client/commit/f5c4561e1db6174a56d6b32b388ed7ad94679fdf))
+
+## [1.98.0](https://github.com/nominal-io/nominal-client/compare/v1.97.0...v1.98.0) (2025-12-04)
+
+
+### Features
+
+* as_files_ingested iterator for dataset files ([#533](https://github.com/nominal-io/nominal-client/issues/533)) ([7afc0f6](https://github.com/nominal-io/nominal-client/commit/7afc0f61d24f930d37d55c8e0840ebd18b8711b6))
+
 ## [1.97.0](https://github.com/nominal-io/nominal-client/compare/v1.96.0...v1.97.0) (2025-12-04)
 
 

{nominal-1.97.0 → nominal-1.99.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nominal
-Version: 1.97.0
+Version: 1.99.0
 Summary: Automate Nominal workflows in Python
 Project-URL: Homepage, https://nominal.io
 Project-URL: Documentation, https://docs.nominal.io

{nominal-1.97.0 → nominal-1.99.0}/nominal/core/__init__.py

@@ -17,7 +17,7 @@ from nominal.core.containerized_extractors import (
 )
 from nominal.core.data_review import CheckViolation, DataReview, DataReviewBuilder
 from nominal.core.dataset import Dataset, poll_until_ingestion_completed
-from nominal.core.dataset_file import DatasetFile
+from nominal.core.dataset_file import DatasetFile, IngestWaitType, as_files_ingested, wait_for_files_to_ingest
 from nominal.core.datasource import DataSource
 from nominal.core.event import Event, EventType
 from nominal.core.filetype import FileType, FileTypes
@@ -33,6 +33,7 @@ from nominal.core.workbook_template import WorkbookTemplate
 from nominal.core.workspace import Workspace
 
 __all__ = [
+    "as_files_ingested",
     "Asset",
     "Attachment",
     "Bounds",
@@ -53,6 +54,7 @@ __all__ = [
     "FileExtractionInput",
     "FileType",
     "FileTypes",
+    "IngestWaitType",
     "LinkDict",
     "LogPoint",
     "NominalClient",
@@ -67,6 +69,7 @@ __all__ = [
     "UserPassAuth",
     "Video",
     "VideoFile",
+    "wait_for_files_to_ingest",
     "Workbook",
     "WorkbookTemplate",
     "WorkbookType",

{nominal-1.97.0 → nominal-1.99.0}/nominal/core/dataset.py

@@ -203,6 +203,78 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
     # Backward compatibility
     add_to_dataset_from_io = add_from_io
 
+    def add_avro_stream(
+        self,
+        path: Path | str,
+    ) -> DatasetFile:
+        """Upload an avro stream file with a specific schema, described below.
+
+        This is a "stream-like" file format to support
+        use cases where a columnar/tabular format does not make sense. This closely matches Nominal's streaming
+        API, making it useful for use cases where network connection drops during streaming and a backup file needs
+        to be created.
+
+        If this schema is not used, will result in a failed ingestion.
+        {
+            "type": "record",
+            "name": "AvroStream",
+            "namespace": "io.nominal.ingest",
+            "fields": [
+                {
+                    "name": "channel",
+                    "type": "string",
+                    "doc": "Channel/series name (e.g., 'vehicle_id', 'col_1', 'temperature')",
+                },
+                {
+                    "name": "timestamps",
+                    "type": {"type": "array", "items": "long"},
+                    "doc": "Array of Unix timestamps in nanoseconds",
+                },
+                {
+                    "name": "values",
+                    "type": {"type": "array", "items": ["double", "string"]},
+                    "doc": "Array of values. Can either be doubles or strings",
+                },
+                {
+                    "name": "tags",
+                    "type": {"type": "map", "values": "string"},
+                    "default": {},
+                    "doc": "Key-value metadata tags",
+                },
+            ],
+        }
+
+        Args:
+            path: Path to the .avro file to upload
+
+        Returns:
+            Reference to the ingesting DatasetFile
+
+        """
+        avro_path = Path(path)
+        s3_path = upload_multipart_file(
+            self._clients.auth_header,
+            self._clients.workspace_rid,
+            avro_path,
+            self._clients.upload,
+            file_type=FileTypes.AVRO_STREAM,
+        )
+        target = ingest_api.DatasetIngestTarget(
+            existing=ingest_api.ExistingDatasetIngestDestination(dataset_rid=self.rid)
+        )
+        resp = self._clients.ingest.ingest(
+            self._clients.auth_header,
+            ingest_api.IngestRequest(
+                options=ingest_api.IngestOptions(
+                    avro_stream=ingest_api.AvroStreamOpts(
+                        source=ingest_api.IngestSource(s3=ingest_api.S3IngestSource(s3_path)),
+                        target=target,
+                    )
+                )
+            ),
+        )
+        return self._handle_ingest_response(resp)
+
     def add_journal_json(
         self,
         path: Path | str,

{nominal-1.97.0 → nominal-1.99.0}/nominal/core/dataset_file.py

@@ -6,7 +6,7 @@ import pathlib
 import time
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Mapping, Protocol, Sequence
+from typing import Iterable, Mapping, Protocol, Sequence
 from urllib.parse import unquote, urlparse
 
 from nominal_api import api, ingest_api, scout_catalog
@@ -287,3 +287,115 @@ class IngestStatus(Enum):
         elif status.error is not None:
             return cls.FAILED
         raise ValueError(f"Unknown ingest status: {status.type}")
+
+
+class IngestWaitType(Enum):
+    FIRST_COMPLETED = "FIRST_COMPLETED"
+    FIRST_EXCEPTION = "FIRST_EXCEPTION"
+    ALL_COMPLETED = "ALL_COMPLETED"
+
+
+def wait_for_files_to_ingest(
+    files: Sequence[DatasetFile],
+    *,
+    poll_interval: datetime.timedelta = datetime.timedelta(seconds=1),
+    timeout: datetime.timedelta | None = None,
+    return_when: IngestWaitType = IngestWaitType.ALL_COMPLETED,
+) -> tuple[Sequence[DatasetFile], Sequence[DatasetFile]]:
+    """Blocks until all of the dataset files have completed their ingestion (or other specified conditions)
+    in a similar fashion to `concurrent.futures.wait`.
+
+    Any files that are already ingested (successfully or with errors) will be returned as "done", whereas any
+    files still ingesting by the time of this function's exit will be returned as "not done".
+
+    Args:
+        files: Dataset files to monitor for ingestion completion.
+        poll_interval: Interval to sleep between polling the remaining files under watch.
+        timeout: If given, the maximum time to wait before returning
+        return_when: Condition for this function to exit. By default, this function will block until all files
+            have completed their ingestion (successfully or unsuccessfully), but this can be changed to return
+            upon the first completed or first failing ingest. This behavior mirrors that of
+            `concurrent.futures.wait`.
+
+    Returns:
+        Returns a tuple of (done, not done) dataset files.
+    """
+    start_time = datetime.datetime.now()
+    done: list[DatasetFile] = []
+    not_done: list[DatasetFile] = [*files]
+    has_failed = False
+
+    while not_done and (timeout is None or datetime.datetime.now() - start_time < timeout):
+        logger.info("Polling for ingestion completion for %d files (%d total)", len(not_done), len(files))
+
+        next_not_done = []
+        for file in not_done:
+            latest_api = file._get_latest_api()
+            latest_file = file._refresh_from_api(latest_api)
+            match file.ingest_status:
+                case IngestStatus.SUCCESS:
+                    done.append(latest_file)
+                case IngestStatus.FAILED:
+                    logger.warning(
+                        "Dataset file %s from dataset %s failed to ingest! Error message: %s",
+                        latest_file.id,
+                        latest_file.dataset_rid,
+                        latest_api.ingest_status.error.message if latest_api.ingest_status.error else "",
+                    )
+                    done.append(latest_file)
+                    has_failed = True
+                case IngestStatus.IN_PROGRESS:
+                    next_not_done.append(latest_file)
+
+        not_done = next_not_done
+
+        if has_failed and return_when is IngestWaitType.FIRST_EXCEPTION:
+            break
+        elif done and return_when is IngestWaitType.FIRST_COMPLETED:
+            break
+        elif not not_done:
+            break
+
+        if timeout is not None and datetime.datetime.now() - start_time < timeout:
+            logger.info(
+                "Sleeping for %f seconds while awaiting ingestion for %d files (%d total)... ",
+                len(not_done),
+                len(files),
+                poll_interval.total_seconds(),
+            )
+            time.sleep(poll_interval.total_seconds())
+
+    return done, not_done
+
+
+def as_files_ingested(
+    files: Sequence[DatasetFile],
+    *,
+    poll_interval: datetime.timedelta = datetime.timedelta(seconds=1),
+) -> Iterable[DatasetFile]:
+    """Iterates over DatasetFiles as they complete their ingestion in a similar fashion to
+    `concurrent.futures.as_completed`.
+
+    Any files that are already ingested (successfully or with errors) will immediately be yielded.
+
+    Args:
+        files: Dataset files to monitor for ingestion completion.
+        poll_interval: Interval to sleep between polling the remaining files under watch.
+
+    Yields:
+        Yields DatasetFiles as they are ingested. Due to the polling mechanics, the files are not yielded in
+        strictly sorted order based on their ingestion completion time. Ensure to check the `ingest_status` of
+        yielded dataset files if important.
+    """
+    to_poll: Sequence[DatasetFile] = [*files]
+    while to_poll:
+        logger.info("Awaiting ingestion for %d files (%d total)", len(to_poll), len(files))
+        done, not_done = wait_for_files_to_ingest(
+            to_poll, poll_interval=poll_interval, return_when=IngestWaitType.FIRST_COMPLETED
+        )
+        for file in done:
+            yield file
+
+        to_poll = not_done
+        if to_poll:
+            time.sleep(poll_interval.total_seconds())

{nominal-1.97.0 → nominal-1.99.0}/nominal/core/filetype.py

@@ -111,6 +111,7 @@ class FileType(NamedTuple):
 
 
 class FileTypes:
+    AVRO_STREAM: FileType = FileType(".avro", "application/avro")
     BINARY: FileType = FileType("", "application/octet-stream")
     CSV: FileType = FileType(".csv", "text/csv")
     CSV_GZ: FileType = FileType(".csv.gz", "text/csv")

{nominal-1.97.0 → nominal-1.99.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nominal"
-version = "1.97.0"
+version = "1.99.0"
 description = "Automate Nominal workflows in Python"
 authors = [
     { name = "Alexander Reynolds", email = "alex.reynolds@nominal.io" },