cognite-extractor-utils 7.5.7__py3-none-any.whl → 7.5.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-extractor-utils might be problematic. Click here for more details.
- cognite/extractorutils/__init__.py +1 -1
- cognite/extractorutils/_inner_util.py +1 -1
- cognite/extractorutils/base.py +4 -3
- cognite/extractorutils/configtools/_util.py +2 -1
- cognite/extractorutils/configtools/elements.py +1 -1
- cognite/extractorutils/configtools/loaders.py +10 -9
- cognite/extractorutils/exceptions.py +1 -1
- cognite/extractorutils/metrics.py +7 -6
- cognite/extractorutils/statestore/hashing.py +6 -6
- cognite/extractorutils/statestore/watermark.py +13 -13
- cognite/extractorutils/threading.py +1 -1
- cognite/extractorutils/unstable/configuration/exceptions.py +2 -5
- cognite/extractorutils/unstable/configuration/loaders.py +8 -8
- cognite/extractorutils/unstable/configuration/models.py +12 -12
- cognite/extractorutils/unstable/core/base.py +6 -9
- cognite/extractorutils/unstable/core/errors.py +1 -1
- cognite/extractorutils/unstable/core/restart_policy.py +1 -1
- cognite/extractorutils/unstable/core/runtime.py +10 -55
- cognite/extractorutils/unstable/core/tasks.py +1 -1
- cognite/extractorutils/unstable/scheduling/_scheduler.py +1 -1
- cognite/extractorutils/uploader/_base.py +2 -1
- cognite/extractorutils/uploader/assets.py +3 -2
- cognite/extractorutils/uploader/data_modeling.py +3 -2
- cognite/extractorutils/uploader/events.py +2 -2
- cognite/extractorutils/uploader/files.py +13 -18
- cognite/extractorutils/uploader/raw.py +3 -2
- cognite/extractorutils/uploader/time_series.py +9 -8
- cognite/extractorutils/uploader/upload_failure_handler.py +2 -2
- cognite/extractorutils/uploader_extractor.py +7 -6
- cognite/extractorutils/uploader_types.py +2 -1
- cognite/extractorutils/util.py +9 -8
- {cognite_extractor_utils-7.5.7.dist-info → cognite_extractor_utils-7.5.8.dist-info}/METADATA +30 -36
- cognite_extractor_utils-7.5.8.dist-info/RECORD +49 -0
- {cognite_extractor_utils-7.5.7.dist-info → cognite_extractor_utils-7.5.8.dist-info}/WHEEL +1 -1
- cognite_extractor_utils-7.5.7.dist-info/RECORD +0 -49
- {cognite_extractor_utils-7.5.7.dist-info → cognite_extractor_utils-7.5.8.dist-info/licenses}/LICENSE +0 -0
|
@@ -5,8 +5,7 @@ import time
|
|
|
5
5
|
from argparse import ArgumentParser, Namespace
|
|
6
6
|
from multiprocessing import Process, Queue
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Any, Generic,
|
|
9
|
-
from uuid import uuid4
|
|
8
|
+
from typing import Any, Generic, TypeVar
|
|
10
9
|
|
|
11
10
|
from requests.exceptions import ConnectionError
|
|
12
11
|
from typing_extensions import assert_never
|
|
@@ -17,7 +16,6 @@ from cognite.extractorutils.unstable.configuration.exceptions import InvalidConf
|
|
|
17
16
|
from cognite.extractorutils.unstable.configuration.loaders import load_file, load_from_cdf
|
|
18
17
|
from cognite.extractorutils.unstable.configuration.models import ConnectionConfig
|
|
19
18
|
from cognite.extractorutils.unstable.core._dto import Error
|
|
20
|
-
from cognite.extractorutils.util import now
|
|
21
19
|
|
|
22
20
|
from ._messaging import RuntimeMessage
|
|
23
21
|
from .base import ConfigRevision, ConfigType, Extractor, FullConfig
|
|
@@ -30,7 +28,7 @@ ExtractorType = TypeVar("ExtractorType", bound=Extractor)
|
|
|
30
28
|
class Runtime(Generic[ExtractorType]):
|
|
31
29
|
def __init__(
|
|
32
30
|
self,
|
|
33
|
-
extractor:
|
|
31
|
+
extractor: type[ExtractorType],
|
|
34
32
|
) -> None:
|
|
35
33
|
self._extractor_class = extractor
|
|
36
34
|
self._cancellation_token = CancellationToken()
|
|
@@ -127,15 +125,13 @@ class Runtime(Generic[ExtractorType]):
|
|
|
127
125
|
self,
|
|
128
126
|
args: Namespace,
|
|
129
127
|
connection_config: ConnectionConfig,
|
|
130
|
-
) -> tuple[ConfigType, ConfigRevision
|
|
128
|
+
) -> tuple[ConfigType, ConfigRevision]:
|
|
131
129
|
current_config_revision: ConfigRevision
|
|
132
|
-
newest_config_revision: ConfigRevision
|
|
133
130
|
|
|
134
131
|
if args.local_override:
|
|
135
132
|
self.logger.info("Loading local application config")
|
|
136
133
|
|
|
137
134
|
current_config_revision = "local"
|
|
138
|
-
newest_config_revision = "local"
|
|
139
135
|
try:
|
|
140
136
|
application_config = load_file(args.local_override[0], self._extractor_class.CONFIG_TYPE)
|
|
141
137
|
except InvalidConfigError as e:
|
|
@@ -153,50 +149,12 @@ class Runtime(Generic[ExtractorType]):
|
|
|
153
149
|
|
|
154
150
|
errors: list[Error] = []
|
|
155
151
|
|
|
156
|
-
revision: int | None = None
|
|
157
152
|
try:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
self._extractor_class.CONFIG_TYPE,
|
|
164
|
-
revision=revision,
|
|
165
|
-
)
|
|
166
|
-
break
|
|
167
|
-
|
|
168
|
-
except InvalidConfigError as e:
|
|
169
|
-
if e.attempted_revision is None:
|
|
170
|
-
# Should never happen, attempted_revision is set in every handler in load_from_cdf, but it's
|
|
171
|
-
# needed for type checks to pass
|
|
172
|
-
raise e
|
|
173
|
-
|
|
174
|
-
self.logger.error(f"Revision {e.attempted_revision} is invalid: {e.message}")
|
|
175
|
-
|
|
176
|
-
t = now()
|
|
177
|
-
errors.append(
|
|
178
|
-
Error(
|
|
179
|
-
external_id=str(uuid4()),
|
|
180
|
-
level="error",
|
|
181
|
-
description=f"Revision {e.attempted_revision} is invalid",
|
|
182
|
-
details=e.message,
|
|
183
|
-
start_time=t,
|
|
184
|
-
end_time=t,
|
|
185
|
-
task=None,
|
|
186
|
-
)
|
|
187
|
-
)
|
|
188
|
-
|
|
189
|
-
if revision is None:
|
|
190
|
-
revision = e.attempted_revision - 1
|
|
191
|
-
newest_config_revision = e.attempted_revision
|
|
192
|
-
else:
|
|
193
|
-
revision -= 1
|
|
194
|
-
|
|
195
|
-
if revision > 0:
|
|
196
|
-
self.logger.info(f"Falling back to revision {revision}")
|
|
197
|
-
else:
|
|
198
|
-
self.logger.critical("No more revisions to fall back to")
|
|
199
|
-
raise e
|
|
153
|
+
application_config, current_config_revision = load_from_cdf(
|
|
154
|
+
client,
|
|
155
|
+
connection_config.integration,
|
|
156
|
+
self._extractor_class.CONFIG_TYPE,
|
|
157
|
+
)
|
|
200
158
|
|
|
201
159
|
finally:
|
|
202
160
|
if errors:
|
|
@@ -209,7 +167,7 @@ class Runtime(Generic[ExtractorType]):
|
|
|
209
167
|
headers={"cdf-version": "alpha"},
|
|
210
168
|
)
|
|
211
169
|
|
|
212
|
-
return application_config, current_config_revision
|
|
170
|
+
return application_config, current_config_revision
|
|
213
171
|
|
|
214
172
|
def _verify_connection_config(self, connection_config: ConnectionConfig) -> bool:
|
|
215
173
|
client = connection_config.get_cognite_client(
|
|
@@ -281,9 +239,7 @@ class Runtime(Generic[ExtractorType]):
|
|
|
281
239
|
application_config: Any
|
|
282
240
|
while not self._cancellation_token.is_cancelled:
|
|
283
241
|
try:
|
|
284
|
-
application_config, current_config_revision
|
|
285
|
-
args, connection_config
|
|
286
|
-
)
|
|
242
|
+
application_config, current_config_revision = self._get_application_config(args, connection_config)
|
|
287
243
|
|
|
288
244
|
except InvalidConfigError:
|
|
289
245
|
self.logger.critical("Could not get a valid application config file. Shutting down")
|
|
@@ -295,7 +251,6 @@ class Runtime(Generic[ExtractorType]):
|
|
|
295
251
|
connection_config=connection_config,
|
|
296
252
|
application_config=application_config,
|
|
297
253
|
current_config_revision=current_config_revision,
|
|
298
|
-
newest_config_revision=newest_config_revision,
|
|
299
254
|
)
|
|
300
255
|
)
|
|
301
256
|
process.join()
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from collections.abc import Callable
|
|
15
16
|
from types import TracebackType
|
|
16
|
-
from typing import Any
|
|
17
|
+
from typing import Any
|
|
17
18
|
|
|
18
19
|
from cognite.client import CogniteClient
|
|
19
20
|
from cognite.client.data_classes.assets import Asset
|
|
@@ -147,7 +148,7 @@ class AssetUploadQueue(AbstractUploadQueue):
|
|
|
147
148
|
|
|
148
149
|
def __exit__(
|
|
149
150
|
self,
|
|
150
|
-
exc_type:
|
|
151
|
+
exc_type: type[BaseException] | None,
|
|
151
152
|
exc_val: BaseException | None,
|
|
152
153
|
exc_tb: TracebackType | None,
|
|
153
154
|
) -> None:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
1
2
|
from types import TracebackType
|
|
2
|
-
from typing import Any
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
from cognite.client import CogniteClient
|
|
5
6
|
from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
|
|
@@ -100,7 +101,7 @@ class InstanceUploadQueue(AbstractUploadQueue):
|
|
|
100
101
|
|
|
101
102
|
def __exit__(
|
|
102
103
|
self,
|
|
103
|
-
exc_type:
|
|
104
|
+
exc_type: type[BaseException] | None,
|
|
104
105
|
exc_val: BaseException | None,
|
|
105
106
|
exc_tb: TracebackType | None,
|
|
106
107
|
) -> None:
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from collections.abc import Callable
|
|
15
16
|
from types import TracebackType
|
|
16
|
-
from typing import Callable, Type
|
|
17
17
|
|
|
18
18
|
from cognite.client import CogniteClient
|
|
19
19
|
from cognite.client.data_classes import Event
|
|
@@ -151,7 +151,7 @@ class EventUploadQueue(AbstractUploadQueue):
|
|
|
151
151
|
return self
|
|
152
152
|
|
|
153
153
|
def __exit__(
|
|
154
|
-
self, exc_type:
|
|
154
|
+
self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
155
155
|
) -> None:
|
|
156
156
|
"""
|
|
157
157
|
Wraps around stop method, for use as context manager
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import threading
|
|
16
|
+
from collections.abc import Callable, Iterator
|
|
16
17
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
17
18
|
from io import BytesIO, RawIOBase
|
|
18
19
|
from math import ceil
|
|
@@ -21,12 +22,6 @@ from types import TracebackType
|
|
|
21
22
|
from typing import (
|
|
22
23
|
Any,
|
|
23
24
|
BinaryIO,
|
|
24
|
-
Callable,
|
|
25
|
-
Iterator,
|
|
26
|
-
List,
|
|
27
|
-
Optional,
|
|
28
|
-
Type,
|
|
29
|
-
Union,
|
|
30
25
|
)
|
|
31
26
|
from urllib.parse import ParseResult, urlparse
|
|
32
27
|
|
|
@@ -67,7 +62,7 @@ _MAX_FILE_CHUNK_SIZE = 4 * 1024 * 1024 * 1000
|
|
|
67
62
|
_CDF_ALPHA_VERSION_HEADER = {"cdf-version": "alpha"}
|
|
68
63
|
|
|
69
64
|
|
|
70
|
-
FileMetadataOrCogniteExtractorFile =
|
|
65
|
+
FileMetadataOrCogniteExtractorFile = FileMetadata | CogniteExtractorFileApply
|
|
71
66
|
|
|
72
67
|
|
|
73
68
|
class ChunkedStream(RawIOBase, BinaryIO):
|
|
@@ -111,7 +106,7 @@ class ChunkedStream(RawIOBase, BinaryIO):
|
|
|
111
106
|
|
|
112
107
|
def __exit__(
|
|
113
108
|
self,
|
|
114
|
-
exc_type:
|
|
109
|
+
exc_type: type[BaseException] | None,
|
|
115
110
|
exc_val: BaseException | None,
|
|
116
111
|
exc_tb: TracebackType | None,
|
|
117
112
|
) -> None:
|
|
@@ -146,7 +141,7 @@ class ChunkedStream(RawIOBase, BinaryIO):
|
|
|
146
141
|
if size > 0:
|
|
147
142
|
self._pos += size
|
|
148
143
|
return self._inner.read(size)
|
|
149
|
-
return
|
|
144
|
+
return b""
|
|
150
145
|
|
|
151
146
|
def next_chunk(self) -> bool:
|
|
152
147
|
"""
|
|
@@ -209,8 +204,8 @@ class IOFileUploadQueue(AbstractUploadQueue):
|
|
|
209
204
|
trigger_log_level: str = "DEBUG",
|
|
210
205
|
thread_name: str | None = None,
|
|
211
206
|
overwrite_existing: bool = False,
|
|
212
|
-
cancellation_token:
|
|
213
|
-
max_parallelism:
|
|
207
|
+
cancellation_token: CancellationToken | None = None,
|
|
208
|
+
max_parallelism: int | None = None,
|
|
214
209
|
failure_logging_path: None | str = None,
|
|
215
210
|
ssl_verify: bool | str = True,
|
|
216
211
|
):
|
|
@@ -231,8 +226,8 @@ class IOFileUploadQueue(AbstractUploadQueue):
|
|
|
231
226
|
self.failure_logging_path = failure_logging_path or None
|
|
232
227
|
self.initialize_failure_logging()
|
|
233
228
|
|
|
234
|
-
self.upload_queue:
|
|
235
|
-
self.errors:
|
|
229
|
+
self.upload_queue: list[Future] = []
|
|
230
|
+
self.errors: list[Exception] = []
|
|
236
231
|
|
|
237
232
|
self.overwrite_existing = overwrite_existing
|
|
238
233
|
|
|
@@ -429,7 +424,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
|
|
|
429
424
|
self,
|
|
430
425
|
file_meta: FileMetadataOrCogniteExtractorFile,
|
|
431
426
|
read_file: Callable[[], BinaryIO],
|
|
432
|
-
extra_retries: tuple[
|
|
427
|
+
extra_retries: tuple[type[Exception], ...] | dict[type[Exception], Callable[[Any], bool]] | None = None,
|
|
433
428
|
) -> None:
|
|
434
429
|
"""
|
|
435
430
|
Add file to upload queue. The file will start uploading immedeately. If the size of the queue is larger than
|
|
@@ -584,9 +579,9 @@ class IOFileUploadQueue(AbstractUploadQueue):
|
|
|
584
579
|
|
|
585
580
|
def __exit__(
|
|
586
581
|
self,
|
|
587
|
-
exc_type:
|
|
588
|
-
exc_val:
|
|
589
|
-
exc_tb:
|
|
582
|
+
exc_type: type[BaseException] | None,
|
|
583
|
+
exc_val: BaseException | None,
|
|
584
|
+
exc_tb: TracebackType | None,
|
|
590
585
|
) -> None:
|
|
591
586
|
"""
|
|
592
587
|
Wraps around stop method, for use as context manager
|
|
@@ -649,7 +644,7 @@ class FileUploadQueue(IOFileUploadQueue):
|
|
|
649
644
|
def add_to_upload_queue(
|
|
650
645
|
self,
|
|
651
646
|
file_meta: FileMetadataOrCogniteExtractorFile,
|
|
652
|
-
file_name:
|
|
647
|
+
file_name: str | PathLike,
|
|
653
648
|
) -> None:
|
|
654
649
|
"""
|
|
655
650
|
Add file to upload queue. The queue will be uploaded if the queue size is larger than the threshold
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from collections.abc import Callable
|
|
15
16
|
from types import TracebackType
|
|
16
|
-
from typing import Any
|
|
17
|
+
from typing import Any
|
|
17
18
|
|
|
18
19
|
import arrow
|
|
19
20
|
from arrow import Arrow
|
|
@@ -162,7 +163,7 @@ class RawUploadQueue(AbstractUploadQueue):
|
|
|
162
163
|
return self
|
|
163
164
|
|
|
164
165
|
def __exit__(
|
|
165
|
-
self, exc_type:
|
|
166
|
+
self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
166
167
|
) -> None:
|
|
167
168
|
"""
|
|
168
169
|
Wraps around stop method, for use as context manager
|
|
@@ -13,9 +13,10 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import math
|
|
16
|
+
from collections.abc import Callable
|
|
16
17
|
from datetime import datetime
|
|
17
18
|
from types import TracebackType
|
|
18
|
-
from typing import Any
|
|
19
|
+
from typing import Any
|
|
19
20
|
|
|
20
21
|
from cognite.client import CogniteClient
|
|
21
22
|
from cognite.client.data_classes import (
|
|
@@ -317,7 +318,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
|
|
|
317
318
|
return self
|
|
318
319
|
|
|
319
320
|
def __exit__(
|
|
320
|
-
self, exc_type:
|
|
321
|
+
self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
321
322
|
) -> None:
|
|
322
323
|
"""
|
|
323
324
|
Wraps around stop method, for use as context manager
|
|
@@ -474,11 +475,11 @@ class SequenceUploadQueue(AbstractUploadQueue):
|
|
|
474
475
|
if isinstance(rows, SequenceRows):
|
|
475
476
|
# Already in the desired format
|
|
476
477
|
pass
|
|
477
|
-
elif isinstance(rows,
|
|
478
|
+
elif isinstance(rows, dict | list):
|
|
478
479
|
rows_raw: list[dict[str, Any]]
|
|
479
480
|
if isinstance(rows, dict):
|
|
480
481
|
rows_raw = [{"rowNumber": row_number, "values": values} for row_number, values in rows.items()]
|
|
481
|
-
elif isinstance(rows, list) and rows and isinstance(rows[0],
|
|
482
|
+
elif isinstance(rows, list) and rows and isinstance(rows[0], tuple | list):
|
|
482
483
|
rows_raw = [{"rowNumber": row_number, "values": values} for row_number, values in rows]
|
|
483
484
|
else:
|
|
484
485
|
rows_raw = rows # type: ignore[assignment]
|
|
@@ -491,7 +492,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
|
|
|
491
492
|
}
|
|
492
493
|
)
|
|
493
494
|
else:
|
|
494
|
-
raise TypeError("Unsupported type for sequence rows: {
|
|
495
|
+
raise TypeError(f"Unsupported type for sequence rows: {type(rows)}")
|
|
495
496
|
|
|
496
497
|
with self.lock:
|
|
497
498
|
seq = self.upload_queue.get(either_id)
|
|
@@ -520,7 +521,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
|
|
|
520
521
|
backoff=RETRY_BACKOFF_FACTOR,
|
|
521
522
|
)
|
|
522
523
|
def _upload_single(either_id: EitherId, upload_this: SequenceData) -> SequenceData:
|
|
523
|
-
self.logger.debug("Writing {} rows to sequence {}"
|
|
524
|
+
self.logger.debug(f"Writing {len(upload_this.values)} rows to sequence {either_id}")
|
|
524
525
|
|
|
525
526
|
try:
|
|
526
527
|
self.cdf_client.sequences.data.insert(
|
|
@@ -595,7 +596,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
|
|
|
595
596
|
)
|
|
596
597
|
|
|
597
598
|
except CogniteDuplicatedError:
|
|
598
|
-
self.logger.info("Sequnce already exist: {}"
|
|
599
|
+
self.logger.info(f"Sequnce already exist: {either_id}")
|
|
599
600
|
seq = self.cdf_client.sequences.retrieve( # type: ignore [assignment]
|
|
600
601
|
id=either_id.internal_id,
|
|
601
602
|
external_id=either_id.external_id,
|
|
@@ -656,7 +657,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
|
|
|
656
657
|
return self
|
|
657
658
|
|
|
658
659
|
def __exit__(
|
|
659
|
-
self, exc_type:
|
|
660
|
+
self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
660
661
|
) -> None:
|
|
661
662
|
"""
|
|
662
663
|
Wraps around stop method, for use as context manager
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
from collections.abc import Iterator
|
|
1
2
|
from datetime import datetime
|
|
2
|
-
from typing import Iterator, List
|
|
3
3
|
|
|
4
4
|
import jsonlines
|
|
5
5
|
|
|
@@ -9,7 +9,7 @@ class FileErrorMapping:
|
|
|
9
9
|
self.file_name = file_name
|
|
10
10
|
self.error_reason = error_reason
|
|
11
11
|
|
|
12
|
-
def __iter__(self) -> Iterator[
|
|
12
|
+
def __iter__(self) -> Iterator[list[str]]:
|
|
13
13
|
return iter([[self.file_name, self.error_reason]])
|
|
14
14
|
|
|
15
15
|
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
A module containing a slightly more advanced base extractor class, sorting a generic output into upload queues.
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
|
+
from collections.abc import Callable, Iterable
|
|
19
20
|
from dataclasses import dataclass
|
|
20
21
|
from types import TracebackType
|
|
21
|
-
from typing import Any,
|
|
22
|
+
from typing import Any, TypeVar
|
|
22
23
|
|
|
23
24
|
from more_itertools import peekable
|
|
24
25
|
|
|
@@ -80,7 +81,7 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
80
81
|
description: str,
|
|
81
82
|
version: str | None = None,
|
|
82
83
|
run_handle: RunHandle | None = None,
|
|
83
|
-
config_class:
|
|
84
|
+
config_class: type[UploaderExtractorConfigClass],
|
|
84
85
|
metrics: BaseMetrics | None = None,
|
|
85
86
|
use_default_state_store: bool = True,
|
|
86
87
|
cancellation_token: CancellationToken | None = None,
|
|
@@ -90,7 +91,7 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
90
91
|
handle_interrupts: bool = True,
|
|
91
92
|
middleware: list[Callable[[dict], dict]] | None = None,
|
|
92
93
|
):
|
|
93
|
-
super(
|
|
94
|
+
super().__init__(
|
|
94
95
|
name=name,
|
|
95
96
|
description=description,
|
|
96
97
|
version=version,
|
|
@@ -144,7 +145,7 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
144
145
|
return item
|
|
145
146
|
|
|
146
147
|
def __enter__(self) -> "UploaderExtractor":
|
|
147
|
-
super(
|
|
148
|
+
super().__enter__()
|
|
148
149
|
|
|
149
150
|
queue_config = self.config.queues if self.config.queues else QueueConfigClass()
|
|
150
151
|
self.event_queue = EventUploadQueue(
|
|
@@ -170,9 +171,9 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
170
171
|
return self
|
|
171
172
|
|
|
172
173
|
def __exit__(
|
|
173
|
-
self, exc_type:
|
|
174
|
+
self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
174
175
|
) -> bool:
|
|
175
176
|
self.event_queue.__exit__(exc_type, exc_val, exc_tb)
|
|
176
177
|
self.raw_queue.__exit__(exc_type, exc_val, exc_tb)
|
|
177
178
|
self.time_series_queue.__exit__(exc_type, exc_val, exc_tb)
|
|
178
|
-
return super(
|
|
179
|
+
return super().__exit__(exc_type, exc_val, exc_tb)
|
cognite/extractorutils/util.py
CHANGED
|
@@ -20,12 +20,13 @@ extractors.
|
|
|
20
20
|
import io
|
|
21
21
|
import logging
|
|
22
22
|
import random
|
|
23
|
+
from collections.abc import Callable, Generator, Iterable
|
|
23
24
|
from datetime import datetime, timezone
|
|
24
25
|
from functools import partial, wraps
|
|
25
26
|
from io import RawIOBase
|
|
26
27
|
from threading import Thread
|
|
27
28
|
from time import time
|
|
28
|
-
from typing import Any,
|
|
29
|
+
from typing import Any, TypeVar
|
|
29
30
|
|
|
30
31
|
from decorator import decorator
|
|
31
32
|
|
|
@@ -157,7 +158,7 @@ class EitherId:
|
|
|
157
158
|
Returns:
|
|
158
159
|
A string rep of the EitherId
|
|
159
160
|
"""
|
|
160
|
-
return "{
|
|
161
|
+
return f"{self.type()}: {self.content()}"
|
|
161
162
|
|
|
162
163
|
def __repr__(self) -> str:
|
|
163
164
|
"""
|
|
@@ -313,7 +314,7 @@ _T2 = TypeVar("_T2")
|
|
|
313
314
|
def _retry_internal(
|
|
314
315
|
f: Callable[..., _T2],
|
|
315
316
|
cancellation_token: CancellationToken,
|
|
316
|
-
exceptions: tuple[
|
|
317
|
+
exceptions: tuple[type[Exception], ...] | dict[type[Exception], Callable[[Exception], bool]],
|
|
317
318
|
tries: int,
|
|
318
319
|
delay: float,
|
|
319
320
|
max_delay: float | None,
|
|
@@ -367,7 +368,7 @@ def _retry_internal(
|
|
|
367
368
|
|
|
368
369
|
def retry(
|
|
369
370
|
cancellation_token: CancellationToken | None = None,
|
|
370
|
-
exceptions: tuple[
|
|
371
|
+
exceptions: tuple[type[Exception], ...] | dict[type[Exception], Callable[[Any], bool]] = (Exception,),
|
|
371
372
|
tries: int = 10,
|
|
372
373
|
delay: float = 1,
|
|
373
374
|
max_delay: float | None = 60,
|
|
@@ -415,7 +416,7 @@ def retry(
|
|
|
415
416
|
|
|
416
417
|
def requests_exceptions(
|
|
417
418
|
status_codes: list[int] | None = None,
|
|
418
|
-
) -> dict[
|
|
419
|
+
) -> dict[type[Exception], Callable[[Any], bool]]:
|
|
419
420
|
"""
|
|
420
421
|
Retry exceptions from using the ``requests`` library. This will retry all connection and HTTP errors matching
|
|
421
422
|
the given status codes.
|
|
@@ -449,7 +450,7 @@ def requests_exceptions(
|
|
|
449
450
|
|
|
450
451
|
def httpx_exceptions(
|
|
451
452
|
status_codes: list[int] | None = None,
|
|
452
|
-
) -> dict[
|
|
453
|
+
) -> dict[type[Exception], Callable[[Any], bool]]:
|
|
453
454
|
"""
|
|
454
455
|
Retry exceptions from using the ``httpx`` library. This will retry all connection and HTTP errors matching
|
|
455
456
|
the given status codes.
|
|
@@ -483,7 +484,7 @@ def httpx_exceptions(
|
|
|
483
484
|
|
|
484
485
|
def cognite_exceptions(
|
|
485
486
|
status_codes: list[int] | None = None,
|
|
486
|
-
) -> dict[
|
|
487
|
+
) -> dict[type[Exception], Callable[[Any], bool]]:
|
|
487
488
|
"""
|
|
488
489
|
Retry exceptions from using the Cognite SDK. This will retry all connection and HTTP errors matching
|
|
489
490
|
the given status codes.
|
|
@@ -499,7 +500,7 @@ def cognite_exceptions(
|
|
|
499
500
|
status_codes = status_codes or [408, 425, 429, 500, 502, 503, 504]
|
|
500
501
|
|
|
501
502
|
def handle_cognite_errors(exception: CogniteException) -> bool:
|
|
502
|
-
if isinstance(exception,
|
|
503
|
+
if isinstance(exception, CogniteAPIError | CogniteFileUploadError):
|
|
503
504
|
return exception.code in status_codes
|
|
504
505
|
return True
|
|
505
506
|
|
{cognite_extractor_utils-7.5.7.dist-info → cognite_extractor_utils-7.5.8.dist-info}/METADATA
RENAMED
|
@@ -1,39 +1,34 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: cognite-extractor-utils
|
|
3
|
-
Version: 7.5.
|
|
3
|
+
Version: 7.5.8
|
|
4
4
|
Summary: Utilities for easier development of extractors for CDF
|
|
5
|
-
|
|
5
|
+
Project-URL: repository, https://github.com/cognitedata/python-extractor-utils
|
|
6
|
+
Author-email: Mathias Lohne <mathias.lohne@cognite.com>
|
|
6
7
|
License: Apache-2.0
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: arrow>=1.0.0
|
|
13
|
+
Requires-Dist: azure-identity>=1.14.0
|
|
14
|
+
Requires-Dist: azure-keyvault-secrets>=4.7.0
|
|
15
|
+
Requires-Dist: cognite-sdk>=7.59.0
|
|
16
|
+
Requires-Dist: croniter>=6.0.0
|
|
17
|
+
Requires-Dist: dacite>=1.6.0
|
|
18
|
+
Requires-Dist: decorator>=5.1.1
|
|
19
|
+
Requires-Dist: httpx<1,>=0.27.0
|
|
20
|
+
Requires-Dist: jsonlines>=4.0.0
|
|
21
|
+
Requires-Dist: more-itertools>=10.0.0
|
|
22
|
+
Requires-Dist: orjson>=3.10.3
|
|
23
|
+
Requires-Dist: prometheus-client<=1.0.0,>=0.7.0
|
|
24
|
+
Requires-Dist: psutil>=6.0.0
|
|
25
|
+
Requires-Dist: pydantic>=2.8.2
|
|
26
|
+
Requires-Dist: pyhumps>=3.8.0
|
|
27
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
28
|
+
Requires-Dist: pyyaml<7,>=5.3.0
|
|
29
|
+
Requires-Dist: typing-extensions<5,>=3.7.4
|
|
17
30
|
Provides-Extra: experimental
|
|
18
|
-
Requires-Dist:
|
|
19
|
-
Requires-Dist: azure-identity (>=1.14.0,<2.0.0)
|
|
20
|
-
Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0)
|
|
21
|
-
Requires-Dist: cognite-sdk (>=7.59.0,<8.0.0)
|
|
22
|
-
Requires-Dist: croniter (>=6.0.0,<7.0.0)
|
|
23
|
-
Requires-Dist: dacite (>=1.6.0,<2.0.0)
|
|
24
|
-
Requires-Dist: decorator (>=5.1.1,<6.0.0)
|
|
25
|
-
Requires-Dist: httpx (>=0.27.0,<1)
|
|
26
|
-
Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
|
|
27
|
-
Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
|
|
28
|
-
Requires-Dist: orjson (>=3.10.3,<4.0.0)
|
|
29
|
-
Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)
|
|
30
|
-
Requires-Dist: psutil (>=6.0.0,<7.0.0)
|
|
31
|
-
Requires-Dist: pydantic (>=2.8.2,<3.0.0)
|
|
32
|
-
Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
|
|
33
|
-
Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
|
|
34
|
-
Requires-Dist: pyyaml (>=5.3.0,<7)
|
|
35
|
-
Requires-Dist: typing-extensions (>=3.7.4,<5)
|
|
36
|
-
Project-URL: Repository, https://github.com/cognitedata/python-extractor-utils
|
|
31
|
+
Requires-Dist: cognite-sdk-experimental; extra == 'experimental'
|
|
37
32
|
Description-Content-Type: text/markdown
|
|
38
33
|
|
|
39
34
|
<a href="https://cognite.com/">
|
|
@@ -96,10 +91,10 @@ as a code of conduct.
|
|
|
96
91
|
|
|
97
92
|
### Development environment
|
|
98
93
|
|
|
99
|
-
We use [
|
|
94
|
+
We use [uv](https://docs.astral.sh/uv/) to manage dependencies and to administrate virtual environments. To develop
|
|
100
95
|
`extractor-utils`, follow the following steps to set up your local environment:
|
|
101
96
|
|
|
102
|
-
1. [Install
|
|
97
|
+
1. [Install uv](https://docs.astral.sh/uv/getting-started/installation/) if you haven't already.
|
|
103
98
|
|
|
104
99
|
2. Clone repository:
|
|
105
100
|
```
|
|
@@ -111,7 +106,7 @@ We use [poetry](https://python-poetry.org) to manage dependencies and to adminis
|
|
|
111
106
|
```
|
|
112
107
|
4. Create virtual environment and install dependencies:
|
|
113
108
|
```
|
|
114
|
-
$
|
|
109
|
+
$ uv sync
|
|
115
110
|
```
|
|
116
111
|
|
|
117
112
|
|
|
@@ -127,4 +122,3 @@ $ poetry run pre-commit install
|
|
|
127
122
|
Each public method, class and module should have docstrings. Docstrings are written in the [Google
|
|
128
123
|
style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). Please include unit and/or
|
|
129
124
|
integration tests for submitted code, and remember to update the [changelog](./CHANGELOG.md).
|
|
130
|
-
|