nominal 1.96.0__py3-none-any.whl → 1.98.0__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [1.98.0](https://github.com/nominal-io/nominal-client/compare/v1.97.0...v1.98.0) (2025-12-04)
+
+
+### Features
+
+* as_files_ingested iterator for dataset files ([#533](https://github.com/nominal-io/nominal-client/issues/533)) ([7afc0f6](https://github.com/nominal-io/nominal-client/commit/7afc0f61d24f930d37d55c8e0840ebd18b8711b6))
+
+## [1.97.0](https://github.com/nominal-io/nominal-client/compare/v1.96.0...v1.97.0) (2025-12-04)
+
+
+### Features
+
+* add create_workbook_template ([#535](https://github.com/nominal-io/nominal-client/issues/535)) ([9c98975](https://github.com/nominal-io/nominal-client/commit/9c989753828c39e417bafed3db2981444132aeac))
+
+
+### Bug Fixes
+
+* add per-part retries during multipart uploads ([#537](https://github.com/nominal-io/nominal-client/issues/537)) ([ffdbc4a](https://github.com/nominal-io/nominal-client/commit/ffdbc4ac6f82562fb9eff3fed4b254b784aa89bc))
+
 ## [1.96.0](https://github.com/nominal-io/nominal-client/compare/v1.95.0...v1.96.0) (2025-12-03)
 
 

nominal/core/__init__.py

@@ -17,7 +17,7 @@ from nominal.core.containerized_extractors import (
 )
 from nominal.core.data_review import CheckViolation, DataReview, DataReviewBuilder
 from nominal.core.dataset import Dataset, poll_until_ingestion_completed
-from nominal.core.dataset_file import DatasetFile
+from nominal.core.dataset_file import DatasetFile, IngestWaitType, as_files_ingested, wait_for_files_to_ingest
 from nominal.core.datasource import DataSource
 from nominal.core.event import Event, EventType
 from nominal.core.filetype import FileType, FileTypes
@@ -33,6 +33,7 @@ from nominal.core.workbook_template import WorkbookTemplate
 from nominal.core.workspace import Workspace
 
 __all__ = [
+    "as_files_ingested",
     "Asset",
     "Attachment",
     "Bounds",
@@ -53,6 +54,7 @@ __all__ = [
     "FileExtractionInput",
     "FileType",
     "FileTypes",
+    "IngestWaitType",
     "LinkDict",
     "LogPoint",
     "NominalClient",
@@ -67,6 +69,7 @@ __all__ = [
     "UserPassAuth",
     "Video",
     "VideoFile",
+    "wait_for_files_to_ingest",
     "Workbook",
     "WorkbookTemplate",
     "WorkbookType",

nominal/core/_utils/multipart.py

@@ -27,29 +27,53 @@ def _sign_and_upload_part_job(
     upload_id: str,
     q: Queue[bytes],
     part: int,
+    num_retries: int = 3,
 ) -> requests.Response:
     data = q.get()
+
     try:
-        response = upload_client.sign_part(auth_header, key, part, upload_id)
-        logger.debug(
-            "successfully signed multipart upload part",
-            extra={"key": key, "part": part, "upload_id": upload_id, "response.url": response.url},
-        )
-        put_response = requests.put(
-            response.url,
-            data=data,
-            headers=response.headers,
-            verify=upload_client._verify,
-        )
-        logger.debug(
-            "put multipart upload part",
-            extra={"url": response.url, "size": len(data), "status_code": put_response.status_code},
+        last_ex: Exception | None = None
+        for attempt in range(num_retries):
+            try:
+                log_extras = {"key": key, "part": part, "upload_id": upload_id, "attempt": attempt + 1}
+
+                logger.debug("Signing part %d for upload", part, extra=log_extras)
+                sign_response = upload_client.sign_part(auth_header, key, part, upload_id)
+                logger.debug(
+                    "Successfully signed part %d for upload",
+                    part,
+                    extra={"response.url": sign_response.url, **log_extras},
+                )
+
+                logger.debug("Pushing part %d for multipart upload", extra=log_extras)
+                put_response = requests.put(
+                    sign_response.url,
+                    data=data,
+                    headers=sign_response.headers,
+                    verify=upload_client._verify,
+                )
+                logger.debug(
+                    "Finished pushing part %d for multipart upload with status %d",
+                    part,
+                    put_response.status_code,
+                    extra={"response.url": put_response.url, **log_extras},
+                )
+                put_response.raise_for_status()
+                return put_response
+            except Exception as ex:
+                logger.warning(
+                    "Failed to upload part %d: %s",
+                    part,
+                    ex,
+                    extra=log_extras,
+                )
+                last_ex = ex
+
+        raise (
+            last_ex
+            if last_ex
+            else RuntimeError(f"Unknown error uploading part {part} for upload_id={upload_id} and key={key}")
         )
-        put_response.raise_for_status()
-        return put_response
-    except Exception as e:
-        logger.exception("error uploading part", exc_info=e, extra={"key": key, "upload_id": upload_id, "part": part})
-        raise e
     finally:
         q.task_done()
 

nominal/core/client.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import enum
 import logging
+import uuid
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from io import TextIOBase
@@ -21,10 +22,12 @@ from nominal_api import (
     scout_catalog,
     scout_checks_api,
     scout_datasource_connection_api,
+    scout_layout_api,
     scout_notebook_api,
     scout_run_api,
     scout_template_api,
     scout_video_api,
+    scout_workbookcommon_api,
     secrets_api,
     storage_datasource_api,
 )
@@ -1478,3 +1481,52 @@ class NominalClient:
             "nominal.core.NominalClient.create_workbook_from_template",
             "use 'nominal.core.WorkbookTemplate.create_workbook' instead",
         )
+
+    def create_workbook_template(
+        self,
+        title: str,
+        *,
+        description: str | None = None,
+        labels: list[str] | None = None,
+        properties: dict[str, str] | None = None,
+        commit_message: str | None = None,
+        workspace: Workspace | str | None = None,
+    ) -> WorkbookTemplate:
+        """Create an empty workbook template.
+
+        Args:
+            title: Title of the workbook template
+            description: Description of the workbook template
+            labels: Labels to attach to the workbook template
+            properties: Properties to attach to the workbook template
+            commit_message: An optional message to include with the creation of the template
+            workspace: Workspace to create the workbook template in.
+
+        Returns:
+            The created WorkbookTemplate
+        """
+        request = scout_template_api.CreateTemplateRequest(
+            title=title,
+            description=description if description is not None else "",
+            labels=labels if labels is not None else [],
+            properties=properties if properties is not None else {},
+            is_published=False,
+            layout=scout_layout_api.WorkbookLayout(
+                v1=scout_layout_api.WorkbookLayoutV1(
+                    root_panel=scout_layout_api.Panel(
+                        tabbed=scout_layout_api.TabbedPanel(
+                            v1=scout_layout_api.TabbedPanelV1(
+                                id=str(uuid.uuid4()),
+                                tabs=[],
+                            )
+                        )
+                    )
+                )
+            ),
+            content=scout_workbookcommon_api.WorkbookContent(channel_variables={}, charts={}),
+            message=commit_message if commit_message is not None else "Initial blank workbook template",
+            workspace=self._workspace_rid_for_search(workspace or WorkspaceSearchType.ALL),
+        )
+
+        template = self._clients.template.create(self._clients.auth_header, request)
+        return WorkbookTemplate._from_conjure(self._clients, template)

nominal/core/dataset_file.py

@@ -6,7 +6,7 @@ import pathlib
 import time
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Mapping, Protocol, Sequence
+from typing import Iterable, Mapping, Protocol, Sequence
 from urllib.parse import unquote, urlparse
 
 from nominal_api import api, ingest_api, scout_catalog
@@ -287,3 +287,115 @@ class IngestStatus(Enum):
         elif status.error is not None:
             return cls.FAILED
         raise ValueError(f"Unknown ingest status: {status.type}")
+
+
+class IngestWaitType(Enum):
+    FIRST_COMPLETED = "FIRST_COMPLETED"
+    FIRST_EXCEPTION = "FIRST_EXCEPTION"
+    ALL_COMPLETED = "ALL_COMPLETED"
+
+
+def wait_for_files_to_ingest(
+    files: Sequence[DatasetFile],
+    *,
+    poll_interval: datetime.timedelta = datetime.timedelta(seconds=1),
+    timeout: datetime.timedelta | None = None,
+    return_when: IngestWaitType = IngestWaitType.ALL_COMPLETED,
+) -> tuple[Sequence[DatasetFile], Sequence[DatasetFile]]:
+    """Blocks until all of the dataset files have completed their ingestion (or other specified conditions)
+    in a similar fashion to `concurrent.futures.wait`.
+
+    Any files that are already ingested (successfully or with errors) will be returned as "done", whereas any
+    files still ingesting by the time of this function's exit will be returned as "not done".
+
+    Args:
+        files: Dataset files to monitor for ingestion completion.
+        poll_interval: Interval to sleep between polling the remaining files under watch.
+        timeout: If given, the maximum time to wait before returning
+        return_when: Condition for this function to exit. By default, this function will block until all files
+            have completed their ingestion (successfully or unsuccessfully), but this can be changed to return
+            upon the first completed or first failing ingest. This behavior mirrors that of
+            `concurrent.futures.wait`.
+
+    Returns:
+        Returns a tuple of (done, not done) dataset files.
+    """
+    start_time = datetime.datetime.now()
+    done: list[DatasetFile] = []
+    not_done: list[DatasetFile] = [*files]
+    has_failed = False
+
+    while not_done and (timeout is None or datetime.datetime.now() - start_time < timeout):
+        logger.info("Polling for ingestion completion for %d files (%d total)", len(not_done), len(files))
+
+        next_not_done = []
+        for file in not_done:
+            latest_api = file._get_latest_api()
+            latest_file = file._refresh_from_api(latest_api)
+            match file.ingest_status:
+                case IngestStatus.SUCCESS:
+                    done.append(latest_file)
+                case IngestStatus.FAILED:
+                    logger.warning(
+                        "Dataset file %s from dataset %s failed to ingest! Error message: %s",
+                        latest_file.id,
+                        latest_file.dataset_rid,
+                        latest_api.ingest_status.error.message if latest_api.ingest_status.error else "",
+                    )
+                    done.append(latest_file)
+                    has_failed = True
+                case IngestStatus.IN_PROGRESS:
+                    next_not_done.append(latest_file)
+
+        not_done = next_not_done
+
+        if has_failed and return_when is IngestWaitType.FIRST_EXCEPTION:
+            break
+        elif done and return_when is IngestWaitType.FIRST_COMPLETED:
+            break
+        elif not not_done:
+            break
+
+        if timeout is not None and datetime.datetime.now() - start_time < timeout:
+            logger.info(
+                "Sleeping for %f seconds while awaiting ingestion for %d files (%d total)... ",
+                len(not_done),
+                len(files),
+                poll_interval.total_seconds(),
+            )
+            time.sleep(poll_interval.total_seconds())
+
+    return done, not_done
+
+
+def as_files_ingested(
+    files: Sequence[DatasetFile],
+    *,
+    poll_interval: datetime.timedelta = datetime.timedelta(seconds=1),
+) -> Iterable[DatasetFile]:
+    """Iterates over DatasetFiles as they complete their ingestion in a similar fashion to
+    `concurrent.futures.as_completed`.
+
+    Any files that are already ingested (successfully or with errors) will immediately be yielded.
+
+    Args:
+        files: Dataset files to monitor for ingestion completion.
+        poll_interval: Interval to sleep between polling the remaining files under watch.
+
+    Yields:
+        Yields DatasetFiles as they are ingested. Due to the polling mechanics, the files are not yielded in
+        strictly sorted order based on their ingestion completion time. Ensure to check the `ingest_status` of
+        yielded dataset files if important.
+    """
+    to_poll: Sequence[DatasetFile] = [*files]
+    while to_poll:
+        logger.info("Awaiting ingestion for %d files (%d total)", len(to_poll), len(files))
+        done, not_done = wait_for_files_to_ingest(
+            to_poll, poll_interval=poll_interval, return_when=IngestWaitType.FIRST_COMPLETED
+        )
+        for file in done:
+            yield file
+
+        to_poll = not_done
+        if to_poll:
+            time.sleep(poll_interval.total_seconds())

{nominal-1.96.0.dist-info → nominal-1.98.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nominal
-Version: 1.96.0
+Version: 1.98.0
 Summary: Automate Nominal workflows in Python
 Project-URL: Homepage, https://nominal.io
 Project-URL: Documentation, https://docs.nominal.io

{nominal-1.96.0.dist-info → nominal-1.98.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=NZ7acl4CWsmucCYmnmZR2SPA6dRL2FekSVJtIWfAkEY,79763
+CHANGELOG.md,sha256=gqkvZQ_HzybFgggDMLGPXEt7eyGDq6jDf26HGFVlKLY,80650
 LICENSE,sha256=zEGHG9mjDjaIS3I79O8mweQo-yiTbqx8jJvUPppVAwk,1067
 README.md,sha256=KKe0dxh_pHXCtB7I9G4qWGQYvot_BZU8yW6MJyuyUHM,311
 nominal/__init__.py,sha256=rbraORnXUrNn1hywLXM0XwSQCd9UmQt20PDYlsBalfE,2167
@@ -27,7 +27,7 @@ nominal/cli/util/global_decorators.py,sha256=SBxhz4KbMlWDcCV08feouftd3HLnBNR-JJt
 nominal/cli/util/verify_connection.py,sha256=KU17ejaDfKBLmLiZ3MZSVLyfrqNE7c6mFBvskhqQLCo,1902
 nominal/config/__init__.py,sha256=wV8cq8X3J4NTJ5H_uR5THaMT_NQpWQO5qCUGEb-rPnM,3157
 nominal/config/_config.py,sha256=yKq_H1iYJDoxRfLz2iXLbbVdoL0MTEY0FS4eVL12w0g,2004
-nominal/core/__init__.py,sha256=N2N_kesiFkvfD6eVmGus2gwAfCsZbwR9KzYw4Wtlvc8,2193
+nominal/core/__init__.py,sha256=5eC2J0lzpV7JcuKDUimJCfgXuVL7HNgHrLhqxcy5NCc,2333
 nominal/core/_clientsbunch.py,sha256=YwciugX7rQ9AOPHyvKuavG7b9SlX1PURRquP37nvLqE,8458
 nominal/core/_constants.py,sha256=SrxgaSqAEB1MvTSrorgGam3eO29iCmRr6VIdajxX3gI,56
 nominal/core/asset.py,sha256=Kq3RvdFSdAK-ViACpd_-H30fz1lnOaGU3zgFt13ag20,16674
@@ -35,12 +35,12 @@ nominal/core/attachment.py,sha256=iJaDyF6JXsKxxBLA03I0WMmQF8U0bA-wRwvXMEhfWLU,42
 nominal/core/bounds.py,sha256=742BWmGL3FBryRAjoiJRg2N6aVinjYkQLxN7kfnJ40Q,581
 nominal/core/channel.py,sha256=dbe8wpfMiWqHu98x66w6GOmC9Ro33Wv9AhBVx2DvtVk,18970
 nominal/core/checklist.py,sha256=rO1RPDYV3o2miPKF7DcCiYpj6bUN-sdtZNhJkXzkfYE,7110
-nominal/core/client.py,sha256=5raS_n__qbUjpCqqxAovd6LAAxRoSkpL87GhoSCPCj0,65205
+nominal/core/client.py,sha256=L6IQVEPTiKbOjtbn4G0_R90jbVeOOxpHBHmminGQ3FE,67403
 nominal/core/connection.py,sha256=ySbPN_a2takVa8wIU9mK4fB6vYLyZnN-qSmXVkLUxAY,5157
 nominal/core/containerized_extractors.py,sha256=HrcMJzdE-hH66AgYIA0LTeFELsBHa0Sm0vlsKMiIzDU,9501
 nominal/core/data_review.py,sha256=bEnRsd8LI4x9YOBPcF2H3h5-e12A7Gh8gQfsNUAZmPQ,7922
 nominal/core/dataset.py,sha256=gbQYtAYx-fHaewOZUSC7P9CHlMfmdchtTv1XUuNKg3Y,29933
-nominal/core/dataset_file.py,sha256=OhkRsI4F3bz9YLF_lQklFfvi6Crq1r0zMMRGkWQEbeQ,11709
+nominal/core/dataset_file.py,sha256=oENANJ17A4K63cZ8Fr7lUm_kVPyA4fL2rUsZ3oXXk2U,16396
 nominal/core/datasource.py,sha256=D9jHirAzUZ0pc3nW1XIURpw1UqQoA2E-nUUylZR1jbE,16707
 nominal/core/event.py,sha256=D8qIX_dTjfSHN7jFW8vV-9htbQTaqk9VvRfK7t-sbbw,5891
 nominal/core/exceptions.py,sha256=GUpwXRgdYamLl6684FE8ttCRHkBx6WEhOZ3NPE-ybD4,2671
@@ -63,7 +63,7 @@ nominal/core/_stream/write_stream_base.py,sha256=AxK3fAq3IBjNXZkxYFVXu3dGNWLCBhg
 nominal/core/_utils/README.md,sha256=kWPQDc6kn-PjXFUsIH9u2nOA3RdGSXCOlxqeJSmUsPA,160
 nominal/core/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nominal/core/_utils/api_tools.py,sha256=Z--Et7NjpCH4if72WwGm45EyqyeqK2ambcBrtOSDMrY,2949
-nominal/core/_utils/multipart.py,sha256=6bce0kZB510TovD71HnjP-EJr6nas9vfO0VPZB4LVY0,9104
+nominal/core/_utils/multipart.py,sha256=YZb6SDJG9eM9AmSHikMbkEOtxy1DTOi9ggVUVzO0Ur4,9998
 nominal/core/_utils/multipart_downloader.py,sha256=zS6wxecAZYeWBfiOjtscoUVPaSiUYTI_Ckn4sknSxMM,14275
 nominal/core/_utils/networking.py,sha256=0cted8IF52WdYAPtiAy9IeosmQkoytXOnazJkdYRkk8,5862
 nominal/core/_utils/pagination_tools.py,sha256=cEBY1WiA1d3cWJEM0myYF_pX8JdQ_e-5asngVXrUc_Y,12152
@@ -102,8 +102,8 @@ nominal/thirdparty/polars/polars_export_handler.py,sha256=hGCSwXX9dC4MG01CmmjlTb
 nominal/thirdparty/tdms/__init__.py,sha256=6n2ImFr2Wiil6JM1P5Q7Mpr0VzLcnDkmup_ftNpPq-s,142
 nominal/thirdparty/tdms/_tdms.py,sha256=eiHFTUviyDPDClckNldjs_jTTSH_sdmboKDq0oIGChQ,8711
 nominal/ts/__init__.py,sha256=hmd0ENvDhxRnzDKGLxIub6QG8LpcxCgcyAct029CaEs,21442
-nominal-1.96.0.dist-info/METADATA,sha256=jq3jplT23Y5wxFaRyMjVL0znHbpFavQSqG6yt00xhyE,1946
-nominal-1.96.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-nominal-1.96.0.dist-info/entry_points.txt,sha256=-mCLhxgg9R_lm5efT7vW9wuBH12izvY322R0a3TYxbE,66
-nominal-1.96.0.dist-info/licenses/LICENSE,sha256=zEGHG9mjDjaIS3I79O8mweQo-yiTbqx8jJvUPppVAwk,1067
-nominal-1.96.0.dist-info/RECORD,,
+nominal-1.98.0.dist-info/METADATA,sha256=av9CrAtGvlkkUC6pdW1oC7TazjZD1tJNIN46C4G4Jng,1946
+nominal-1.98.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+nominal-1.98.0.dist-info/entry_points.txt,sha256=-mCLhxgg9R_lm5efT7vW9wuBH12izvY322R0a3TYxbE,66
+nominal-1.98.0.dist-info/licenses/LICENSE,sha256=zEGHG9mjDjaIS3I79O8mweQo-yiTbqx8jJvUPppVAwk,1067
+nominal-1.98.0.dist-info/RECORD,,