nominal 1.99.0__tar.gz → 1.100.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nominal-1.99.0 → nominal-1.100.0}/CHANGELOG.md +19 -0
- {nominal-1.99.0 → nominal-1.100.0}/PKG-INFO +3 -2
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/multipart.py +8 -3
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/multipart_downloader.py +2 -16
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/networking.py +72 -7
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/asset.py +43 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/client.py +4 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/containerized_extractors.py +22 -3
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/dataset.py +49 -2
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/run.py +6 -8
- nominal-1.100.0/nominal/experimental/migration/__init__.py +19 -0
- nominal-1.100.0/nominal/experimental/migration/migration_utils.py +552 -0
- {nominal-1.99.0 → nominal-1.100.0}/pyproject.toml +3 -2
- {nominal-1.99.0 → nominal-1.100.0}/.gitignore +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/LICENSE +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/README.md +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/__main__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/README.md +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/dataclass_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/deprecation_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/iterator_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/streaming_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/_utils/timing_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/__main__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/attachment.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/auth.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/config.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/dataset.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/download.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/mis.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/run.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/util/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/util/click_log_handler.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/util/global_decorators.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/cli/util/verify_connection.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/config/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/config/_config.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_clientsbunch.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_constants.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_stream/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_stream/batch_processor.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_stream/batch_processor_proto.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_stream/write_stream.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_stream/write_stream_base.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/README.md +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/api_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/pagination_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/query_tools.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/_utils/queueing.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/attachment.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/bounds.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/channel.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/checklist.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/connection.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/data_review.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/dataset_file.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/datasource.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/event.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/exceptions.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/filetype.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/log.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/secret.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/unit.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/user.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/video.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/video_file.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/workbook.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/workbook_template.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/core/workspace.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/exceptions/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/README.md +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/_buckets.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/_enum_expr_impls.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/_numeric_expr_impls.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/_range_expr_impls.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/exprs.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/compute/dsl/params.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/logging/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/logging/click_log_handler.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/logging/nominal_log_handler.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/logging/rich_log_handler.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/rust_streaming/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/rust_streaming/rust_write_stream.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/stream_v2/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/stream_v2/_serializer.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/stream_v2/_write_stream.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/video_processing/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/video_processing/resolution.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/experimental/video_processing/video_conversion.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/nominal.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/py.typed +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/matlab/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/matlab/_matlab.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/pandas/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/pandas/_pandas.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/polars/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/polars/polars_export_handler.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/tdms/__init__.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/thirdparty/tdms/_tdms.py +0 -0
- {nominal-1.99.0 → nominal-1.100.0}/nominal/ts/__init__.py +0 -0
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.100.0](https://github.com/nominal-io/nominal-client/compare/v1.99.0...v1.100.0) (2025-12-19)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* add args and timestamp types for containerized ingest ([#551](https://github.com/nominal-io/nominal-client/issues/551)) ([1dd259a](https://github.com/nominal-io/nominal-client/commit/1dd259a299d83c55b482f443542f7feb63896795))
|
|
9
|
+
* add clone/copy_from methods for assets, datasets, templates ([#548](https://github.com/nominal-io/nominal-client/issues/548)) ([f04c468](https://github.com/nominal-io/nominal-client/commit/f04c46800569da0ad6fb9440a336a7d8cac3542a))
|
|
10
|
+
* added prefix_tree_delimiter parameter to get_or_create_dataset method ([#550](https://github.com/nominal-io/nominal-client/issues/550)) ([c204d94](https://github.com/nominal-io/nominal-client/commit/c204d9450d33a9742c0ae6b6a2d78e8baa796d2c))
|
|
11
|
+
* allow listing runs on asset, deprecate search run by asset in client ([#541](https://github.com/nominal-io/nominal-client/issues/541)) ([35464e5](https://github.com/nominal-io/nominal-client/commit/35464e56ebc81577094b4ce0862d830e7d7bc92e))
|
|
12
|
+
* allow promoting assets ([#542](https://github.com/nominal-io/nominal-client/issues/542)) ([1ce1082](https://github.com/nominal-io/nominal-client/commit/1ce1082dd025371bed4920c0e10d5ec93ac37687))
|
|
13
|
+
* allow using truststore for ssl bypass ([#472](https://github.com/nominal-io/nominal-client/issues/472)) ([55a43c2](https://github.com/nominal-io/nominal-client/commit/55a43c23cb5a32b6cf90b2d50c5f0e79eeafaca7))
|
|
14
|
+
* create new clone workbook and associated helpers in experimental ([#546](https://github.com/nominal-io/nominal-client/issues/546)) ([aeffb44](https://github.com/nominal-io/nominal-client/commit/aeffb44827b46e90101404d4a9a18d036aac49ab))
|
|
15
|
+
* expose truststore.SSLContext across requests usage to permit usage in corporate networks ([55a43c2](https://github.com/nominal-io/nominal-client/commit/55a43c23cb5a32b6cf90b2d50c5f0e79eeafaca7))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Bug Fixes
|
|
19
|
+
|
|
20
|
+
* fix log message missing argument ([#547](https://github.com/nominal-io/nominal-client/issues/547)) ([c684caa](https://github.com/nominal-io/nominal-client/commit/c684caa559544f7c5c2f13e6d94fd8f2e718827c))
|
|
21
|
+
|
|
3
22
|
## [1.99.0](https://github.com/nominal-io/nominal-client/compare/v1.98.0...v1.99.0) (2025-12-04)
|
|
4
23
|
|
|
5
24
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nominal
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.100.0
|
|
4
4
|
Summary: Automate Nominal workflows in Python
|
|
5
5
|
Project-URL: Homepage, https://nominal.io
|
|
6
6
|
Project-URL: Documentation, https://docs.nominal.io
|
|
@@ -20,7 +20,7 @@ Requires-Dist: cachetools>=6.1.0
|
|
|
20
20
|
Requires-Dist: click<9,>=8
|
|
21
21
|
Requires-Dist: conjure-python-client<4,>=3.1.0
|
|
22
22
|
Requires-Dist: ffmpeg-python>=0.2.0
|
|
23
|
-
Requires-Dist: nominal-api==0.
|
|
23
|
+
Requires-Dist: nominal-api==0.1032.0
|
|
24
24
|
Requires-Dist: nominal-streaming==0.5.8
|
|
25
25
|
Requires-Dist: openpyxl>=0.0.0
|
|
26
26
|
Requires-Dist: pandas>=0.0.0
|
|
@@ -30,6 +30,7 @@ Requires-Dist: pyyaml>=0.0.0
|
|
|
30
30
|
Requires-Dist: requests>=0.0.0
|
|
31
31
|
Requires-Dist: rich>=14.1.0
|
|
32
32
|
Requires-Dist: tabulate<0.10,>=0.9.0
|
|
33
|
+
Requires-Dist: truststore>=0.10.4
|
|
33
34
|
Requires-Dist: types-cachetools>=6.0.0.20250525
|
|
34
35
|
Requires-Dist: typing-extensions<5,>=4
|
|
35
36
|
Provides-Extra: protos
|
|
@@ -11,6 +11,7 @@ from typing import BinaryIO, Iterable
|
|
|
11
11
|
import requests
|
|
12
12
|
from nominal_api import ingest_api, upload_api
|
|
13
13
|
|
|
14
|
+
from nominal.core._utils.networking import create_multipart_request_session
|
|
14
15
|
from nominal.core.exceptions import NominalMultipartUploadFailed
|
|
15
16
|
from nominal.core.filetype import FileType
|
|
16
17
|
|
|
@@ -22,6 +23,7 @@ DEFAULT_NUM_WORKERS = 8
|
|
|
22
23
|
|
|
23
24
|
def _sign_and_upload_part_job(
|
|
24
25
|
upload_client: upload_api.UploadService,
|
|
26
|
+
multipart_session: requests.Session,
|
|
25
27
|
auth_header: str,
|
|
26
28
|
key: str,
|
|
27
29
|
upload_id: str,
|
|
@@ -45,8 +47,8 @@ def _sign_and_upload_part_job(
|
|
|
45
47
|
extra={"response.url": sign_response.url, **log_extras},
|
|
46
48
|
)
|
|
47
49
|
|
|
48
|
-
logger.debug("Pushing part %d for multipart upload", extra=log_extras)
|
|
49
|
-
put_response =
|
|
50
|
+
logger.debug("Pushing part %d for multipart upload", part, extra=log_extras)
|
|
51
|
+
put_response = multipart_session.put(
|
|
50
52
|
sign_response.url,
|
|
51
53
|
data=data,
|
|
52
54
|
headers=sign_response.headers,
|
|
@@ -141,7 +143,10 @@ def put_multipart_upload(
|
|
|
141
143
|
)
|
|
142
144
|
initiate_response = upload_client.initiate_multipart_upload(auth_header, initiate_request)
|
|
143
145
|
key, upload_id = initiate_response.key, initiate_response.upload_id
|
|
144
|
-
|
|
146
|
+
multipart_session = create_multipart_request_session(pool_size=max_workers)
|
|
147
|
+
_sign_and_upload_part = partial(
|
|
148
|
+
_sign_and_upload_part_job, upload_client, multipart_session, auth_header, key, upload_id, q
|
|
149
|
+
)
|
|
145
150
|
|
|
146
151
|
jobs: list[concurrent.futures.Future[requests.Response]] = []
|
|
147
152
|
|
|
@@ -13,11 +13,10 @@ from types import TracebackType
|
|
|
13
13
|
from typing import Callable, Iterable, Mapping, Sequence, Type
|
|
14
14
|
|
|
15
15
|
import requests
|
|
16
|
-
from requests.adapters import HTTPAdapter
|
|
17
16
|
from typing_extensions import Self
|
|
18
|
-
from urllib3.util.retry import Retry
|
|
19
17
|
|
|
20
18
|
from nominal.core._utils.multipart import DEFAULT_CHUNK_SIZE
|
|
19
|
+
from nominal.core._utils.networking import create_multipart_request_session
|
|
21
20
|
|
|
22
21
|
logger = logging.getLogger(__name__)
|
|
23
22
|
|
|
@@ -129,25 +128,12 @@ class MultipartFileDownloader:
|
|
|
129
128
|
max_workers = multiprocessing.cpu_count()
|
|
130
129
|
logger.info("Inferring core count as %d", max_workers)
|
|
131
130
|
|
|
132
|
-
session =
|
|
131
|
+
session = create_multipart_request_session(pool_size=max_workers)
|
|
133
132
|
pool = ThreadPoolExecutor(max_workers=max_workers)
|
|
134
133
|
return cls(max_workers, timeout, max_part_retries, _session=session, _pool=pool, _closed=False)
|
|
135
134
|
|
|
136
135
|
# ---- lifecycle ----
|
|
137
136
|
|
|
138
|
-
@staticmethod
|
|
139
|
-
def _make_session(pool_size: int) -> requests.Session:
|
|
140
|
-
retries = Retry(
|
|
141
|
-
total=5,
|
|
142
|
-
backoff_factor=0.5,
|
|
143
|
-
status_forcelist=(429, 500, 502, 503, 504),
|
|
144
|
-
allowed_methods=frozenset(["GET", "HEAD"]),
|
|
145
|
-
)
|
|
146
|
-
s = requests.Session()
|
|
147
|
-
adapter = HTTPAdapter(max_retries=retries, pool_maxsize=pool_size)
|
|
148
|
-
s.mount("https://", adapter)
|
|
149
|
-
return s
|
|
150
|
-
|
|
151
137
|
def close(self) -> None:
|
|
152
138
|
if not self._closed:
|
|
153
139
|
try:
|
|
@@ -1,20 +1,69 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import gzip
|
|
4
|
+
import logging
|
|
4
5
|
import os
|
|
6
|
+
import ssl
|
|
5
7
|
from typing import Any, Callable, Mapping, Type, TypeVar
|
|
6
8
|
|
|
7
9
|
import requests
|
|
10
|
+
import truststore
|
|
8
11
|
from conjure_python_client import ServiceConfiguration
|
|
9
|
-
from conjure_python_client._http.requests_client import
|
|
10
|
-
from requests.adapters import CaseInsensitiveDict
|
|
12
|
+
from conjure_python_client._http.requests_client import KEEP_ALIVE_SOCKET_OPTIONS, RetryWithJitter
|
|
13
|
+
from requests.adapters import DEFAULT_POOLSIZE, CaseInsensitiveDict, HTTPAdapter
|
|
14
|
+
from urllib3.connection import HTTPConnection
|
|
15
|
+
from urllib3.util.retry import Retry
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
11
18
|
|
|
12
19
|
T = TypeVar("T")
|
|
13
20
|
|
|
14
21
|
GZIP_COMPRESSION_LEVEL = 1
|
|
15
22
|
|
|
16
23
|
|
|
17
|
-
class
|
|
24
|
+
class SslBypassRequestsAdapter(HTTPAdapter):
|
|
25
|
+
"""Transport adapter that allows customizing SSL options and forwarding host truststore.
|
|
26
|
+
|
|
27
|
+
NOTE: based on a combination of injecting `truststore.SSLContext` into
|
|
28
|
+
`conjure_python_client._http.requests_client.TransportAdapter`.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
ENABLE_KEEP_ALIVE_ATTR = "_enable_keep_alive"
|
|
32
|
+
__attrs__ = [*HTTPAdapter.__attrs__, ENABLE_KEEP_ALIVE_ATTR]
|
|
33
|
+
|
|
34
|
+
def __init__(self, *args: Any, enable_keep_alive: bool = False, **kwargs: Any):
|
|
35
|
+
self._enable_keep_alive = enable_keep_alive
|
|
36
|
+
super().__init__(*args, **kwargs)
|
|
37
|
+
|
|
38
|
+
def init_poolmanager(
|
|
39
|
+
self,
|
|
40
|
+
connections: int,
|
|
41
|
+
maxsize: int,
|
|
42
|
+
block: bool = False,
|
|
43
|
+
**pool_kwargs: Mapping[str, Any],
|
|
44
|
+
) -> None:
|
|
45
|
+
"""Wrapper around the standard init_poolmanager from HTTPAdapter with modifications
|
|
46
|
+
to support keep-alive settings and injecting SSL context.
|
|
47
|
+
"""
|
|
48
|
+
if self._enable_keep_alive:
|
|
49
|
+
keep_alive_kwargs: dict[str, Any] = {
|
|
50
|
+
"socket_options": [
|
|
51
|
+
*HTTPConnection.default_socket_options,
|
|
52
|
+
*KEEP_ALIVE_SOCKET_OPTIONS,
|
|
53
|
+
]
|
|
54
|
+
}
|
|
55
|
+
pool_kwargs = {**pool_kwargs, **keep_alive_kwargs}
|
|
56
|
+
|
|
57
|
+
pool_kwargs["ssl_context"] = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
58
|
+
|
|
59
|
+
super().init_poolmanager(connections, maxsize, block, **pool_kwargs) # type: ignore[no-untyped-call]
|
|
60
|
+
|
|
61
|
+
def __setstate__(self, state: dict[str, Any]) -> None:
|
|
62
|
+
state[self.ENABLE_KEEP_ALIVE_ATTR] = state.get(self.ENABLE_KEEP_ALIVE_ATTR, False)
|
|
63
|
+
super().__setstate__(state) # type: ignore[misc]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class NominalRequestsAdapter(SslBypassRequestsAdapter):
|
|
18
67
|
"""Adapter used with `requests` library for sending gzip-compressed data.
|
|
19
68
|
|
|
20
69
|
Based on: https://github.com/psf/requests/issues/1753#issuecomment-417806737
|
|
@@ -69,7 +118,7 @@ class GzipRequestsAdapter(TransportAdapter):
|
|
|
69
118
|
return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
|
|
70
119
|
|
|
71
120
|
|
|
72
|
-
def
|
|
121
|
+
def create_conjure_service_client(
|
|
73
122
|
service_class: Type[T],
|
|
74
123
|
user_agent: str,
|
|
75
124
|
service_config: ServiceConfiguration,
|
|
@@ -104,7 +153,7 @@ def create_gzip_service_client(
|
|
|
104
153
|
status_forcelist=[308, 429, 503],
|
|
105
154
|
backoff_factor=float(service_config.backoff_slot_size) / 1000,
|
|
106
155
|
)
|
|
107
|
-
transport_adapter =
|
|
156
|
+
transport_adapter = NominalRequestsAdapter(max_retries=retry)
|
|
108
157
|
# create a session, for shared connection polling, user agent, etc
|
|
109
158
|
session = requests.Session()
|
|
110
159
|
session.headers = CaseInsensitiveDict({"User-Agent": user_agent})
|
|
@@ -131,11 +180,11 @@ def create_conjure_client_factory(
|
|
|
131
180
|
) -> Callable[[Type[T]], T]:
|
|
132
181
|
"""Create factory method for creating conjure clients given the respective conjure service type
|
|
133
182
|
|
|
134
|
-
See `
|
|
183
|
+
See `create_conjure_service_client` for documentation on parameters.
|
|
135
184
|
"""
|
|
136
185
|
|
|
137
186
|
def factory(service_class: Type[T]) -> T:
|
|
138
|
-
return
|
|
187
|
+
return create_conjure_service_client(
|
|
139
188
|
service_class,
|
|
140
189
|
user_agent=user_agent,
|
|
141
190
|
service_config=service_config,
|
|
@@ -143,3 +192,19 @@ def create_conjure_client_factory(
|
|
|
143
192
|
)
|
|
144
193
|
|
|
145
194
|
return factory
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def create_multipart_request_session(
|
|
198
|
+
*,
|
|
199
|
+
pool_size: int = DEFAULT_POOLSIZE,
|
|
200
|
+
num_retries: int = 5,
|
|
201
|
+
) -> requests.Session:
|
|
202
|
+
retries = Retry(
|
|
203
|
+
total=num_retries,
|
|
204
|
+
backoff_factor=0.5,
|
|
205
|
+
status_forcelist=(429, 500, 502, 503, 504),
|
|
206
|
+
)
|
|
207
|
+
session = requests.Session()
|
|
208
|
+
adapter = SslBypassRequestsAdapter(max_retries=retries, pool_maxsize=pool_size)
|
|
209
|
+
session.mount("https://", adapter)
|
|
210
|
+
return session
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import logging
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
from types import MappingProxyType
|
|
5
6
|
from typing import Iterable, Literal, Mapping, Protocol, Sequence, TypeAlias, cast
|
|
6
7
|
|
|
7
8
|
from nominal_api import (
|
|
9
|
+
scout,
|
|
8
10
|
scout_asset_api,
|
|
9
11
|
scout_assets,
|
|
10
12
|
scout_run_api,
|
|
@@ -13,6 +15,7 @@ from typing_extensions import Self
|
|
|
13
15
|
|
|
14
16
|
from nominal.core._clientsbunch import HasScoutParams
|
|
15
17
|
from nominal.core._utils.api_tools import HasRid, Link, RefreshableMixin, create_links, rid_from_instance_or_string
|
|
18
|
+
from nominal.core._utils.pagination_tools import search_runs_by_asset_paginated
|
|
16
19
|
from nominal.core.attachment import Attachment, _iter_get_attachments
|
|
17
20
|
from nominal.core.connection import Connection, _get_connections
|
|
18
21
|
from nominal.core.dataset import Dataset, _create_dataset, _get_datasets
|
|
@@ -22,6 +25,8 @@ from nominal.ts import IntegralNanosecondsUTC, _SecondsNanos
|
|
|
22
25
|
|
|
23
26
|
ScopeType: TypeAlias = Connection | Dataset | Video
|
|
24
27
|
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
25
30
|
|
|
26
31
|
@dataclass(frozen=True)
|
|
27
32
|
class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
@@ -43,6 +48,8 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
43
48
|
):
|
|
44
49
|
@property
|
|
45
50
|
def assets(self) -> scout_assets.AssetService: ...
|
|
51
|
+
@property
|
|
52
|
+
def run(self) -> scout.RunService: ...
|
|
46
53
|
|
|
47
54
|
@property
|
|
48
55
|
def nominal_url(self) -> str:
|
|
@@ -98,6 +105,22 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
98
105
|
if scope.data_source.type.lower() == stype
|
|
99
106
|
}
|
|
100
107
|
|
|
108
|
+
def promote(self) -> Self:
|
|
109
|
+
"""Promote this asset to be a standard, searchable, and displayable asset.
|
|
110
|
+
|
|
111
|
+
This method is only useful for assets that were created implicitly from creating a run directly on a dataset.
|
|
112
|
+
Nothing will happen from calling this method (aside from a logged warning) if called on a non-staged
|
|
113
|
+
asset (e.g. an asset created by create_asset, or an asset that's already been promoted).
|
|
114
|
+
"""
|
|
115
|
+
if self._get_latest_api().is_staged:
|
|
116
|
+
request = scout_asset_api.UpdateAssetRequest(is_staged=False)
|
|
117
|
+
updated_asset = self._clients.assets.update_asset(self._clients.auth_header, request, self.rid)
|
|
118
|
+
self._refresh_from_api(updated_asset)
|
|
119
|
+
else:
|
|
120
|
+
logger.warning("Not promoting asset %s-- already promoted!", self.rid)
|
|
121
|
+
|
|
122
|
+
return self
|
|
123
|
+
|
|
101
124
|
def get_data_scope(self, data_scope_name: str) -> ScopeType:
|
|
102
125
|
"""Retrieve a datascope by data scope name, or raise ValueError if one is not found."""
|
|
103
126
|
for scope, data in self.list_data_scopes():
|
|
@@ -257,6 +280,7 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
257
280
|
description: str | None = None,
|
|
258
281
|
labels: Sequence[str] = (),
|
|
259
282
|
properties: Mapping[str, str] | None = None,
|
|
283
|
+
prefix_tree_delimiter: str | None = None,
|
|
260
284
|
) -> Dataset:
|
|
261
285
|
"""Retrieve a dataset by data scope name, or create a new one if it does not exist."""
|
|
262
286
|
try:
|
|
@@ -272,6 +296,10 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
272
296
|
workspace_rid=self._clients.workspace_rid,
|
|
273
297
|
)
|
|
274
298
|
dataset = Dataset._from_conjure(self._clients, enriched_dataset)
|
|
299
|
+
|
|
300
|
+
if prefix_tree_delimiter is not None:
|
|
301
|
+
dataset.set_channel_prefix_tree(prefix_tree_delimiter)
|
|
302
|
+
|
|
275
303
|
self.add_dataset(data_scope_name, dataset)
|
|
276
304
|
return dataset
|
|
277
305
|
|
|
@@ -372,6 +400,17 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
372
400
|
def list_attachments(self) -> Sequence[Attachment]:
|
|
373
401
|
return list(self._iter_list_attachments())
|
|
374
402
|
|
|
403
|
+
def list_runs(self) -> Sequence[Run]:
|
|
404
|
+
"""List all runs associated with this Asset."""
|
|
405
|
+
return [
|
|
406
|
+
Run._from_conjure(self._clients, run)
|
|
407
|
+
for run in search_runs_by_asset_paginated(
|
|
408
|
+
self._clients.run,
|
|
409
|
+
self._clients.auth_header,
|
|
410
|
+
self.rid,
|
|
411
|
+
)
|
|
412
|
+
]
|
|
413
|
+
|
|
375
414
|
def remove_attachments(self, attachments: Iterable[Attachment] | Iterable[str]) -> None:
|
|
376
415
|
"""Remove attachments from this asset.
|
|
377
416
|
Does not remove the attachments from Nominal.
|
|
@@ -403,3 +442,7 @@ class Asset(HasRid, RefreshableMixin[scout_asset_api.Asset]):
|
|
|
403
442
|
created_at=_SecondsNanos.from_flexible(asset.created_at).to_nanoseconds(),
|
|
404
443
|
_clients=clients,
|
|
405
444
|
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
# Moving to bottom to deal with circular dependencies
|
|
448
|
+
from nominal.core.run import Run # noqa: E402
|
|
@@ -608,6 +608,10 @@ class NominalClient:
|
|
|
608
608
|
)
|
|
609
609
|
)
|
|
610
610
|
|
|
611
|
+
@deprecated(
|
|
612
|
+
"NominalClient.search_runs_by_asset is deprecated and will be removed in a future version. "
|
|
613
|
+
"Use Asset.list_runs() instead."
|
|
614
|
+
)
|
|
611
615
|
def search_runs_by_asset(self, asset: Asset | str) -> Sequence[Run]:
|
|
612
616
|
"""Search for all runs associated with a given asset:
|
|
613
617
|
|
|
@@ -5,7 +5,7 @@ from enum import Enum
|
|
|
5
5
|
from typing import Mapping, Protocol, Sequence
|
|
6
6
|
|
|
7
7
|
from nominal_api import ingest_api
|
|
8
|
-
from typing_extensions import Self
|
|
8
|
+
from typing_extensions import Self, deprecated
|
|
9
9
|
|
|
10
10
|
from nominal._utils.dataclass_tools import update_dataclass
|
|
11
11
|
from nominal.core._clientsbunch import HasScoutParams
|
|
@@ -203,9 +203,22 @@ class ContainerizedExtractor(HasRid):
|
|
|
203
203
|
inputs: Sequence[FileExtractionInput]
|
|
204
204
|
properties: Mapping[str, str]
|
|
205
205
|
labels: Sequence[str]
|
|
206
|
-
|
|
206
|
+
default_timestamp_metadata: TimestampMetadata | None
|
|
207
207
|
_clients: _Clients = field(repr=False)
|
|
208
208
|
|
|
209
|
+
@property
|
|
210
|
+
@deprecated(
|
|
211
|
+
"The `timestamp_metadata` field of a ContainerizedExtractor is deprecated and will be removed in a future "
|
|
212
|
+
"release. Use the `default_timestamp_metadata` field instead."
|
|
213
|
+
)
|
|
214
|
+
def timestamp_metadata(self) -> TimestampMetadata:
|
|
215
|
+
if self.default_timestamp_metadata is None:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Containerized extractor {self.name} ({self.rid}) has no default configured timestamp metadata"
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
return self.default_timestamp_metadata
|
|
221
|
+
|
|
209
222
|
class _Clients(HasScoutParams, Protocol):
|
|
210
223
|
@property
|
|
211
224
|
def containerized_extractors(self) -> ingest_api.ContainerizedExtractorService: ...
|
|
@@ -251,6 +264,12 @@ class ContainerizedExtractor(HasRid):
|
|
|
251
264
|
|
|
252
265
|
@classmethod
|
|
253
266
|
def _from_conjure(cls, clients: _Clients, raw_extractor: ingest_api.ContainerizedExtractor) -> Self:
|
|
267
|
+
timestamp_metadata = (
|
|
268
|
+
None
|
|
269
|
+
if raw_extractor.timestamp_metadata is None
|
|
270
|
+
else TimestampMetadata._from_conjure(raw_extractor.timestamp_metadata)
|
|
271
|
+
)
|
|
272
|
+
|
|
254
273
|
return cls(
|
|
255
274
|
rid=raw_extractor.rid,
|
|
256
275
|
name=raw_extractor.name,
|
|
@@ -259,6 +278,6 @@ class ContainerizedExtractor(HasRid):
|
|
|
259
278
|
inputs=[FileExtractionInput._from_conjure(raw_input) for raw_input in raw_extractor.inputs],
|
|
260
279
|
properties=raw_extractor.properties,
|
|
261
280
|
labels=raw_extractor.labels,
|
|
262
|
-
|
|
281
|
+
default_timestamp_metadata=timestamp_metadata,
|
|
263
282
|
_clients=clients,
|
|
264
283
|
)
|
|
@@ -6,7 +6,7 @@ from datetime import timedelta
|
|
|
6
6
|
from io import TextIOBase
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from types import MappingProxyType
|
|
9
|
-
from typing import BinaryIO, Iterable, Mapping, Sequence, TypeAlias
|
|
9
|
+
from typing import BinaryIO, Iterable, Mapping, Sequence, TypeAlias, overload
|
|
10
10
|
|
|
11
11
|
from nominal_api import api, ingest_api, scout_catalog
|
|
12
12
|
from typing_extensions import Self, deprecated
|
|
@@ -410,11 +410,38 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
|
|
|
410
410
|
# Backward compatibility
|
|
411
411
|
add_ardupilot_dataflash_to_dataset = add_ardupilot_dataflash
|
|
412
412
|
|
|
413
|
+
@overload
|
|
413
414
|
def add_containerized(
|
|
414
415
|
self,
|
|
415
416
|
extractor: str | ContainerizedExtractor,
|
|
416
417
|
sources: Mapping[str, Path | str],
|
|
417
418
|
tag: str | None = None,
|
|
419
|
+
*,
|
|
420
|
+
arguments: Mapping[str, str] | None = None,
|
|
421
|
+
tags: Mapping[str, str] | None = None,
|
|
422
|
+
) -> DatasetFile: ...
|
|
423
|
+
@overload
|
|
424
|
+
def add_containerized(
|
|
425
|
+
self,
|
|
426
|
+
extractor: str | ContainerizedExtractor,
|
|
427
|
+
sources: Mapping[str, Path | str],
|
|
428
|
+
tag: str | None = None,
|
|
429
|
+
*,
|
|
430
|
+
arguments: Mapping[str, str] | None = None,
|
|
431
|
+
tags: Mapping[str, str] | None = None,
|
|
432
|
+
timestamp_column: str,
|
|
433
|
+
timestamp_type: _AnyTimestampType,
|
|
434
|
+
) -> DatasetFile: ...
|
|
435
|
+
def add_containerized(
|
|
436
|
+
self,
|
|
437
|
+
extractor: str | ContainerizedExtractor,
|
|
438
|
+
sources: Mapping[str, Path | str],
|
|
439
|
+
tag: str | None = None,
|
|
440
|
+
*,
|
|
441
|
+
arguments: Mapping[str, str] | None = None,
|
|
442
|
+
tags: Mapping[str, str] | None = None,
|
|
443
|
+
timestamp_column: str | None = None,
|
|
444
|
+
timestamp_type: _AnyTimestampType | None = None,
|
|
418
445
|
) -> DatasetFile:
|
|
419
446
|
"""Add data from proprietary data formats using a pre-registered custom extractor.
|
|
420
447
|
|
|
@@ -424,7 +451,24 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
|
|
|
424
451
|
NOTE: these must match the registered inputs of the containerized extractor exactly
|
|
425
452
|
tag: Tag of the Docker container which hosts the extractor.
|
|
426
453
|
NOTE: if not provided, the default registered docker tag will be used.
|
|
454
|
+
arguments: Mapping of key-value pairs of input arguments to the extractor.
|
|
455
|
+
tags: Key-value pairs of tags to apply to all data ingested from the containerized extractor run.
|
|
456
|
+
timestamp_column: the column in the dataset that contains the timestamp data.
|
|
457
|
+
NOTE: this is applied uniformly to all output files
|
|
458
|
+
NOTE: must be provided with a `timestamp_type` or a ValueError will be raised
|
|
459
|
+
timestamp_type: the type of timestamp data in the dataset.
|
|
460
|
+
NOTE: this is applied uniformly to all output files
|
|
461
|
+
NOTE: must be provided with a `timestamp_column` or a ValueError will be raised
|
|
427
462
|
"""
|
|
463
|
+
timestamp_metadata = None
|
|
464
|
+
if timestamp_column is not None and timestamp_type is not None:
|
|
465
|
+
timestamp_metadata = ingest_api.TimestampMetadata(
|
|
466
|
+
series_name=timestamp_column,
|
|
467
|
+
timestamp_type=_to_typed_timestamp_type(timestamp_type)._to_conjure_ingest_api(),
|
|
468
|
+
)
|
|
469
|
+
elif None in (timestamp_column, timestamp_type):
|
|
470
|
+
raise ValueError("Only one of `timestamp_column` and `timestamp_type` provided!")
|
|
471
|
+
|
|
428
472
|
if isinstance(extractor, str):
|
|
429
473
|
extractor = ContainerizedExtractor._from_conjure(
|
|
430
474
|
self._clients,
|
|
@@ -451,13 +495,14 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
|
|
|
451
495
|
)
|
|
452
496
|
logger.info("Uploaded %s -> %s", source_path, s3_path)
|
|
453
497
|
s3_inputs[source] = s3_path
|
|
498
|
+
|
|
454
499
|
logger.info("Triggering custom extractor %s (tag=%s) with %s", extractor.name, tag, s3_inputs)
|
|
455
500
|
resp = self._clients.ingest.ingest(
|
|
456
501
|
self._clients.auth_header,
|
|
457
502
|
trigger_ingest=ingest_api.IngestRequest(
|
|
458
503
|
options=ingest_api.IngestOptions(
|
|
459
504
|
containerized=ingest_api.ContainerizedOpts(
|
|
460
|
-
arguments={},
|
|
505
|
+
arguments={**(arguments or {})},
|
|
461
506
|
extractor_rid=extractor.rid,
|
|
462
507
|
sources={
|
|
463
508
|
source: ingest_api.IngestSource(s3=ingest_api.S3IngestSource(path=s3_path))
|
|
@@ -467,6 +512,8 @@ class Dataset(DataSource, RefreshableMixin[scout_catalog.EnrichedDataset]):
|
|
|
467
512
|
existing=ingest_api.ExistingDatasetIngestDestination(self.rid)
|
|
468
513
|
),
|
|
469
514
|
tag=tag,
|
|
515
|
+
additional_file_tags={**(tags or {})},
|
|
516
|
+
timestamp_metadata=timestamp_metadata,
|
|
470
517
|
)
|
|
471
518
|
)
|
|
472
519
|
),
|
|
@@ -6,11 +6,11 @@ from types import MappingProxyType
|
|
|
6
6
|
from typing import Iterable, Mapping, Protocol, Sequence, cast
|
|
7
7
|
|
|
8
8
|
from nominal_api import (
|
|
9
|
-
scout,
|
|
10
9
|
scout_run_api,
|
|
11
10
|
)
|
|
12
11
|
from typing_extensions import Self
|
|
13
12
|
|
|
13
|
+
from nominal.core import asset as core_asset
|
|
14
14
|
from nominal.core._clientsbunch import HasScoutParams
|
|
15
15
|
from nominal.core._utils.api_tools import (
|
|
16
16
|
HasRid,
|
|
@@ -20,7 +20,6 @@ from nominal.core._utils.api_tools import (
|
|
|
20
20
|
create_links,
|
|
21
21
|
rid_from_instance_or_string,
|
|
22
22
|
)
|
|
23
|
-
from nominal.core.asset import Asset
|
|
24
23
|
from nominal.core.attachment import Attachment, _iter_get_attachments
|
|
25
24
|
from nominal.core.connection import Connection, _get_connections
|
|
26
25
|
from nominal.core.dataset import Dataset, _get_datasets
|
|
@@ -45,12 +44,11 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run]):
|
|
|
45
44
|
_clients: _Clients = field(repr=False)
|
|
46
45
|
|
|
47
46
|
class _Clients(
|
|
48
|
-
Asset._Clients,
|
|
47
|
+
core_asset.Asset._Clients,
|
|
49
48
|
HasScoutParams,
|
|
50
49
|
Protocol,
|
|
51
50
|
):
|
|
52
|
-
|
|
53
|
-
def run(self) -> scout.RunService: ...
|
|
51
|
+
pass
|
|
54
52
|
|
|
55
53
|
@property
|
|
56
54
|
def nominal_url(self) -> str:
|
|
@@ -304,13 +302,13 @@ class Run(HasRid, RefreshableMixin[scout_run_api.Run]):
|
|
|
304
302
|
"""List a sequence of Attachments associated with this Run."""
|
|
305
303
|
return list(self._iter_list_attachments())
|
|
306
304
|
|
|
307
|
-
def _iter_list_assets(self) -> Iterable[Asset]:
|
|
305
|
+
def _iter_list_assets(self) -> Iterable[core_asset.Asset]:
|
|
308
306
|
run = self._get_latest_api()
|
|
309
307
|
assets = self._clients.assets.get_assets(self._clients.auth_header, run.assets)
|
|
310
308
|
for a in assets.values():
|
|
311
|
-
yield Asset._from_conjure(self._clients, a)
|
|
309
|
+
yield core_asset.Asset._from_conjure(self._clients, a)
|
|
312
310
|
|
|
313
|
-
def list_assets(self) -> Sequence[Asset]:
|
|
311
|
+
def list_assets(self) -> Sequence[core_asset.Asset]:
|
|
314
312
|
"""List assets associated with this run."""
|
|
315
313
|
return list(self._iter_list_assets())
|
|
316
314
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from nominal.experimental.migration.migration_utils import (
|
|
2
|
+
clone_asset,
|
|
3
|
+
clone_dataset,
|
|
4
|
+
clone_workbook_template,
|
|
5
|
+
copy_asset_from,
|
|
6
|
+
copy_dataset_from,
|
|
7
|
+
copy_resources_to_destination_client,
|
|
8
|
+
copy_workbook_template_from,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"clone_asset",
|
|
13
|
+
"clone_dataset",
|
|
14
|
+
"clone_workbook_template",
|
|
15
|
+
"copy_asset_from",
|
|
16
|
+
"copy_dataset_from",
|
|
17
|
+
"copy_resources_to_destination_client",
|
|
18
|
+
"copy_workbook_template_from",
|
|
19
|
+
]
|