cognite-extractor-utils 7.5.4__py3-none-any.whl → 7.5.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-extractor-utils might be problematic. Click here for more details.
- cognite/extractorutils/__init__.py +3 -1
- cognite/extractorutils/_inner_util.py +14 -3
- cognite/extractorutils/base.py +14 -15
- cognite/extractorutils/configtools/__init__.py +25 -0
- cognite/extractorutils/configtools/_util.py +7 -9
- cognite/extractorutils/configtools/elements.py +58 -49
- cognite/extractorutils/configtools/loaders.py +29 -26
- cognite/extractorutils/configtools/validators.py +2 -3
- cognite/extractorutils/exceptions.py +1 -4
- cognite/extractorutils/metrics.py +18 -18
- cognite/extractorutils/statestore/_base.py +3 -4
- cognite/extractorutils/statestore/hashing.py +24 -24
- cognite/extractorutils/statestore/watermark.py +17 -14
- cognite/extractorutils/threading.py +4 -4
- cognite/extractorutils/unstable/configuration/exceptions.py +24 -0
- cognite/extractorutils/unstable/configuration/loaders.py +18 -7
- cognite/extractorutils/unstable/configuration/models.py +25 -3
- cognite/extractorutils/unstable/core/_dto.py +10 -0
- cognite/extractorutils/unstable/core/base.py +179 -29
- cognite/extractorutils/unstable/core/errors.py +72 -0
- cognite/extractorutils/unstable/core/restart_policy.py +29 -0
- cognite/extractorutils/unstable/core/runtime.py +170 -26
- cognite/extractorutils/unstable/core/tasks.py +2 -0
- cognite/extractorutils/unstable/scheduling/_scheduler.py +4 -4
- cognite/extractorutils/uploader/__init__.py +14 -0
- cognite/extractorutils/uploader/_base.py +8 -8
- cognite/extractorutils/uploader/assets.py +15 -9
- cognite/extractorutils/uploader/data_modeling.py +13 -13
- cognite/extractorutils/uploader/events.py +9 -9
- cognite/extractorutils/uploader/files.py +127 -31
- cognite/extractorutils/uploader/raw.py +10 -10
- cognite/extractorutils/uploader/time_series.py +56 -58
- cognite/extractorutils/uploader/upload_failure_handler.py +64 -0
- cognite/extractorutils/uploader_extractor.py +11 -11
- cognite/extractorutils/uploader_types.py +4 -12
- cognite/extractorutils/util.py +21 -23
- {cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.5.dist-info}/METADATA +3 -2
- cognite_extractor_utils-7.5.5.dist-info/RECORD +49 -0
- {cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.5.dist-info}/WHEEL +1 -1
- cognite/extractorutils/unstable/core/__main__.py +0 -31
- cognite_extractor_utils-7.5.4.dist-info/RECORD +0 -46
- {cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.5.dist-info}/LICENSE +0 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from typing import Iterator, List
|
|
3
|
+
|
|
4
|
+
import jsonlines
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FileErrorMapping:
|
|
8
|
+
def __init__(self, file_name: str, error_reason: str) -> None:
|
|
9
|
+
self.file_name = file_name
|
|
10
|
+
self.error_reason = error_reason
|
|
11
|
+
|
|
12
|
+
def __iter__(self) -> Iterator[List[str]]:
|
|
13
|
+
return iter([[self.file_name, self.error_reason]])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class FileFailureManager:
|
|
17
|
+
MAX_QUEUE_SIZE = 500
|
|
18
|
+
START_TIME_KEY = "start_time"
|
|
19
|
+
FILE_REASON_MAP_KEY = "file_error_reason_map"
|
|
20
|
+
|
|
21
|
+
def __init__(self, start_time: str | None = None, path_to_file: str | None = None) -> None:
|
|
22
|
+
self.failure_logs: dict[str, str] = {}
|
|
23
|
+
|
|
24
|
+
self.path_to_failure_log: str = self._pre_process_file_extension(path_to_file)
|
|
25
|
+
self.start_time = start_time or str(datetime.now())
|
|
26
|
+
self._initialize_failure_logs()
|
|
27
|
+
|
|
28
|
+
def _pre_process_file_extension(self, path_to_file: str | None) -> str:
|
|
29
|
+
if path_to_file and not path_to_file.endswith(".jsonl"):
|
|
30
|
+
return path_to_file + ".jsonl"
|
|
31
|
+
return str(path_to_file)
|
|
32
|
+
|
|
33
|
+
def _initialize_failure_logs(self) -> None:
|
|
34
|
+
self.failure_logs = {}
|
|
35
|
+
|
|
36
|
+
def __len__(self) -> int:
|
|
37
|
+
return len(self.failure_logs)
|
|
38
|
+
|
|
39
|
+
def clear(self) -> None:
|
|
40
|
+
self.failure_logs.clear()
|
|
41
|
+
self._initialize_failure_logs()
|
|
42
|
+
|
|
43
|
+
def add(self, file_name: str, error_reason: str) -> None:
|
|
44
|
+
error_file_object = FileErrorMapping(file_name=file_name, error_reason=error_reason)
|
|
45
|
+
error_file_dict = dict(error_file_object)
|
|
46
|
+
|
|
47
|
+
self.failure_logs.update(error_file_dict)
|
|
48
|
+
|
|
49
|
+
if len(self) >= self.MAX_QUEUE_SIZE:
|
|
50
|
+
self.write_to_file()
|
|
51
|
+
|
|
52
|
+
def write_to_file(self) -> None:
|
|
53
|
+
if len(self) == 0:
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
dict_to_write = {
|
|
57
|
+
self.START_TIME_KEY: self.start_time,
|
|
58
|
+
self.FILE_REASON_MAP_KEY: self.failure_logs,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
with jsonlines.open(self.path_to_failure_log, mode="a") as writer:
|
|
62
|
+
writer.write(dict_to_write)
|
|
63
|
+
|
|
64
|
+
self.clear()
|
|
@@ -15,9 +15,10 @@
|
|
|
15
15
|
"""
|
|
16
16
|
A module containing a slightly more advanced base extractor class, sorting a generic output into upload queues.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
from dataclasses import dataclass
|
|
19
20
|
from types import TracebackType
|
|
20
|
-
from typing import Any, Callable, Iterable,
|
|
21
|
+
from typing import Any, Callable, Iterable, Type, TypeVar
|
|
21
22
|
|
|
22
23
|
from more_itertools import peekable
|
|
23
24
|
|
|
@@ -41,10 +42,11 @@ class QueueConfigClass:
|
|
|
41
42
|
|
|
42
43
|
@dataclass
|
|
43
44
|
class UploaderExtractorConfig(BaseConfig):
|
|
44
|
-
queues:
|
|
45
|
+
queues: QueueConfigClass | None
|
|
45
46
|
|
|
46
47
|
|
|
47
48
|
UploaderExtractorConfigClass = TypeVar("UploaderExtractorConfigClass", bound=UploaderExtractorConfig)
|
|
49
|
+
RunHandle = Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, CancellationToken], None]
|
|
48
50
|
|
|
49
51
|
|
|
50
52
|
class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
@@ -76,19 +78,17 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
76
78
|
*,
|
|
77
79
|
name: str,
|
|
78
80
|
description: str,
|
|
79
|
-
version:
|
|
80
|
-
run_handle:
|
|
81
|
-
Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, CancellationToken], None]
|
|
82
|
-
] = None,
|
|
81
|
+
version: str | None = None,
|
|
82
|
+
run_handle: RunHandle | None = None,
|
|
83
83
|
config_class: Type[UploaderExtractorConfigClass],
|
|
84
|
-
metrics:
|
|
84
|
+
metrics: BaseMetrics | None = None,
|
|
85
85
|
use_default_state_store: bool = True,
|
|
86
|
-
cancellation_token:
|
|
87
|
-
config_file_path:
|
|
86
|
+
cancellation_token: CancellationToken | None = None,
|
|
87
|
+
config_file_path: str | None = None,
|
|
88
88
|
continuous_extractor: bool = False,
|
|
89
89
|
heartbeat_waiting_time: int = 600,
|
|
90
90
|
handle_interrupts: bool = True,
|
|
91
|
-
middleware:
|
|
91
|
+
middleware: list[Callable[[dict], dict]] | None = None,
|
|
92
92
|
):
|
|
93
93
|
super(UploaderExtractor, self).__init__(
|
|
94
94
|
name=name,
|
|
@@ -170,7 +170,7 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
|
|
|
170
170
|
return self
|
|
171
171
|
|
|
172
172
|
def __exit__(
|
|
173
|
-
self, exc_type:
|
|
173
|
+
self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
|
|
174
174
|
) -> bool:
|
|
175
175
|
self.event_queue.__exit__(exc_type, exc_val, exc_tb)
|
|
176
176
|
self.raw_queue.__exit__(exc_type, exc_val, exc_tb)
|
|
@@ -1,27 +1,19 @@
|
|
|
1
|
-
import
|
|
2
|
-
from typing import Iterable, List, Optional, Union
|
|
1
|
+
from typing import Iterable, TypeAlias
|
|
3
2
|
|
|
4
3
|
from cognite.client.data_classes import Event as _Event
|
|
5
4
|
from cognite.client.data_classes import Row as _Row
|
|
6
|
-
|
|
7
|
-
if sys.version_info >= (3, 10):
|
|
8
|
-
from typing import TypeAlias
|
|
9
|
-
else:
|
|
10
|
-
from typing_extensions import TypeAlias
|
|
11
|
-
|
|
12
|
-
|
|
13
5
|
from cognite.extractorutils.uploader.time_series import DataPoint
|
|
14
6
|
|
|
15
7
|
|
|
16
8
|
class InsertDatapoints:
|
|
17
|
-
def __init__(self, *, id:
|
|
9
|
+
def __init__(self, *, id: int | None = None, external_id: str | None = None, datapoints: list[DataPoint]):
|
|
18
10
|
self.id = id
|
|
19
11
|
self.external_id = external_id
|
|
20
12
|
self.datapoints = datapoints
|
|
21
13
|
|
|
22
14
|
|
|
23
15
|
class RawRow:
|
|
24
|
-
def __init__(self, db_name: str, table_name: str, row:
|
|
16
|
+
def __init__(self, db_name: str, table_name: str, row: _Row | Iterable[_Row]):
|
|
25
17
|
self.db_name = db_name
|
|
26
18
|
self.table_name = table_name
|
|
27
19
|
if isinstance(row, Iterable):
|
|
@@ -32,4 +24,4 @@ class RawRow:
|
|
|
32
24
|
|
|
33
25
|
Event: TypeAlias = _Event
|
|
34
26
|
|
|
35
|
-
CdfTypes =
|
|
27
|
+
CdfTypes = Event | Iterable[Event] | RawRow | Iterable[RawRow] | InsertDatapoints | Iterable[InsertDatapoints]
|
cognite/extractorutils/util.py
CHANGED
|
@@ -25,7 +25,7 @@ from functools import partial, wraps
|
|
|
25
25
|
from io import RawIOBase
|
|
26
26
|
from threading import Thread
|
|
27
27
|
from time import time
|
|
28
|
-
from typing import Any, Callable,
|
|
28
|
+
from typing import Any, Callable, Generator, Iterable, Type, TypeVar
|
|
29
29
|
|
|
30
30
|
from decorator import decorator
|
|
31
31
|
|
|
@@ -89,7 +89,7 @@ class EitherId:
|
|
|
89
89
|
TypeError: If none of both of id types are set.
|
|
90
90
|
"""
|
|
91
91
|
|
|
92
|
-
def __init__(self, **kwargs:
|
|
92
|
+
def __init__(self, **kwargs: int | str | None):
|
|
93
93
|
internal_id = kwargs.get("id")
|
|
94
94
|
external_id = kwargs.get("externalId") or kwargs.get("external_id")
|
|
95
95
|
|
|
@@ -105,8 +105,8 @@ class EitherId:
|
|
|
105
105
|
if external_id is not None and not isinstance(external_id, str):
|
|
106
106
|
raise TypeError("External IDs must be strings")
|
|
107
107
|
|
|
108
|
-
self.internal_id:
|
|
109
|
-
self.external_id:
|
|
108
|
+
self.internal_id: int | None = internal_id
|
|
109
|
+
self.external_id: str | None = external_id
|
|
110
110
|
|
|
111
111
|
def type(self) -> str:
|
|
112
112
|
"""
|
|
@@ -117,7 +117,7 @@ class EitherId:
|
|
|
117
117
|
"""
|
|
118
118
|
return "id" if self.internal_id is not None else "externalId"
|
|
119
119
|
|
|
120
|
-
def content(self) ->
|
|
120
|
+
def content(self) -> int | str:
|
|
121
121
|
"""
|
|
122
122
|
Get the value of the ID
|
|
123
123
|
|
|
@@ -249,7 +249,7 @@ def add_extraction_pipeline(
|
|
|
249
249
|
##############################
|
|
250
250
|
_logger.info(f"Starting to run function: {input_function.__name__}")
|
|
251
251
|
|
|
252
|
-
heartbeat_thread:
|
|
252
|
+
heartbeat_thread: Thread | None = None
|
|
253
253
|
try:
|
|
254
254
|
heartbeat_thread = Thread(target=heartbeat_loop, name="HeartbeatLoop", daemon=True)
|
|
255
255
|
heartbeat_thread.start()
|
|
@@ -313,12 +313,12 @@ _T2 = TypeVar("_T2")
|
|
|
313
313
|
def _retry_internal(
|
|
314
314
|
f: Callable[..., _T2],
|
|
315
315
|
cancellation_token: CancellationToken,
|
|
316
|
-
exceptions:
|
|
316
|
+
exceptions: tuple[Type[Exception], ...] | dict[Type[Exception], Callable[[Exception], bool]],
|
|
317
317
|
tries: int,
|
|
318
318
|
delay: float,
|
|
319
|
-
max_delay:
|
|
319
|
+
max_delay: float | None,
|
|
320
320
|
backoff: float,
|
|
321
|
-
jitter:
|
|
321
|
+
jitter: float | tuple[float, float],
|
|
322
322
|
) -> _T2:
|
|
323
323
|
logger = logging.getLogger(__name__)
|
|
324
324
|
|
|
@@ -366,13 +366,13 @@ def _retry_internal(
|
|
|
366
366
|
|
|
367
367
|
|
|
368
368
|
def retry(
|
|
369
|
-
cancellation_token:
|
|
370
|
-
exceptions:
|
|
369
|
+
cancellation_token: CancellationToken | None = None,
|
|
370
|
+
exceptions: tuple[Type[Exception], ...] | dict[Type[Exception], Callable[[Any], bool]] = (Exception,),
|
|
371
371
|
tries: int = 10,
|
|
372
372
|
delay: float = 1,
|
|
373
|
-
max_delay:
|
|
373
|
+
max_delay: float | None = 60,
|
|
374
374
|
backoff: float = 2,
|
|
375
|
-
jitter:
|
|
375
|
+
jitter: float | tuple[float, float] = (0, 2),
|
|
376
376
|
) -> Callable[[Callable[..., _T2]], Callable[..., _T2]]:
|
|
377
377
|
"""
|
|
378
378
|
Returns a retry decorator.
|
|
@@ -414,8 +414,8 @@ def retry(
|
|
|
414
414
|
|
|
415
415
|
|
|
416
416
|
def requests_exceptions(
|
|
417
|
-
status_codes:
|
|
418
|
-
) ->
|
|
417
|
+
status_codes: list[int] | None = None,
|
|
418
|
+
) -> dict[Type[Exception], Callable[[Any], bool]]:
|
|
419
419
|
"""
|
|
420
420
|
Retry exceptions from using the ``requests`` library. This will retry all connection and HTTP errors matching
|
|
421
421
|
the given status codes.
|
|
@@ -448,8 +448,8 @@ def requests_exceptions(
|
|
|
448
448
|
|
|
449
449
|
|
|
450
450
|
def httpx_exceptions(
|
|
451
|
-
status_codes:
|
|
452
|
-
) ->
|
|
451
|
+
status_codes: list[int] | None = None,
|
|
452
|
+
) -> dict[Type[Exception], Callable[[Any], bool]]:
|
|
453
453
|
"""
|
|
454
454
|
Retry exceptions from using the ``httpx`` library. This will retry all connection and HTTP errors matching
|
|
455
455
|
the given status codes.
|
|
@@ -482,8 +482,8 @@ def httpx_exceptions(
|
|
|
482
482
|
|
|
483
483
|
|
|
484
484
|
def cognite_exceptions(
|
|
485
|
-
status_codes:
|
|
486
|
-
) ->
|
|
485
|
+
status_codes: list[int] | None = None,
|
|
486
|
+
) -> dict[Type[Exception], Callable[[Any], bool]]:
|
|
487
487
|
"""
|
|
488
488
|
Retry exceptions from using the Cognite SDK. This will retry all connection and HTTP errors matching
|
|
489
489
|
the given status codes.
|
|
@@ -569,9 +569,7 @@ def truncate_byte_len(item: str, ln: int) -> str:
|
|
|
569
569
|
|
|
570
570
|
|
|
571
571
|
class BufferedReadWithLength(io.BufferedReader):
|
|
572
|
-
def __init__(
|
|
573
|
-
self, raw: RawIOBase, buffer_size: int, len: int, on_close: Optional[Callable[[], None]] = None
|
|
574
|
-
) -> None:
|
|
572
|
+
def __init__(self, raw: RawIOBase, buffer_size: int, len: int, on_close: Callable[[], None] | None = None) -> None:
|
|
575
573
|
super().__init__(raw, buffer_size)
|
|
576
574
|
# Do not remove even if it appears to be unused. :P
|
|
577
575
|
# Requests uses this to add the content-length header, which is necessary for writing to files in azure clusters
|
|
@@ -588,7 +586,7 @@ def iterable_to_stream(
|
|
|
588
586
|
iterator: Iterable[bytes],
|
|
589
587
|
file_size_bytes: int,
|
|
590
588
|
buffer_size: int = io.DEFAULT_BUFFER_SIZE,
|
|
591
|
-
on_close:
|
|
589
|
+
on_close: Callable[[], None] | None = None,
|
|
592
590
|
) -> BufferedReadWithLength:
|
|
593
591
|
class ChunkIteratorStream(io.RawIOBase):
|
|
594
592
|
def __init__(self) -> None:
|
{cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.5.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: cognite-extractor-utils
|
|
3
|
-
Version: 7.5.
|
|
3
|
+
Version: 7.5.5
|
|
4
4
|
Summary: Utilities for easier development of extractors for CDF
|
|
5
5
|
Home-page: https://github.com/cognitedata/python-extractor-utils
|
|
6
6
|
License: Apache-2.0
|
|
@@ -23,6 +23,7 @@ Requires-Dist: croniter (>=5.0.0,<6.0.0)
|
|
|
23
23
|
Requires-Dist: dacite (>=1.6.0,<2.0.0)
|
|
24
24
|
Requires-Dist: decorator (>=5.1.1,<6.0.0)
|
|
25
25
|
Requires-Dist: httpx (>=0.27.0,<0.28.0)
|
|
26
|
+
Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
|
|
26
27
|
Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
|
|
27
28
|
Requires-Dist: orjson (>=3.10.3,<4.0.0)
|
|
28
29
|
Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
cognite/extractorutils/__init__.py,sha256=vzJt70lWLihpyT1wBagsTLt-snzoca1rHtDq5vNKV8M,764
|
|
2
|
+
cognite/extractorutils/_inner_util.py,sha256=cdoz9Sl3Wt1IsxiCZlcd913_hKrTCxDRrM_L-Zn1_F8,1800
|
|
3
|
+
cognite/extractorutils/base.py,sha256=pV3xy0Dzt8q9I5DvI-TvmRZXMmSTk8Kk-d0jZWa_ua8,16333
|
|
4
|
+
cognite/extractorutils/configtools/__init__.py,sha256=llNMzHu4yCWx5Kjm8G9IN5Pij8OUaVT_VZuZ2r3JtAA,3616
|
|
5
|
+
cognite/extractorutils/configtools/_util.py,sha256=uXpR8YnEkfeZOuaZGjRRk_wgC5AGOEKNWMYfV50atsc,4746
|
|
6
|
+
cognite/extractorutils/configtools/elements.py,sha256=ti3PFmwHyiFJFXNEzObRY6IxQo18LABSsYafPxuoYSU,26590
|
|
7
|
+
cognite/extractorutils/configtools/loaders.py,sha256=w8NoZcZJZbEctvkTq8aG_UH2x2gct_fpb2KenksmVaQ,18294
|
|
8
|
+
cognite/extractorutils/configtools/validators.py,sha256=xug3GOMIO4NOdyyvXtYlpKyq9wuDtGf7-xqIefD5bIo,1016
|
|
9
|
+
cognite/extractorutils/exceptions.py,sha256=NDmiElg1cmGMwIl82kpCDF37UcAFNnfDK9NxUn_u2rk,1149
|
|
10
|
+
cognite/extractorutils/metrics.py,sha256=-sUBaZ7lNrcdxuQcsh7rU-CwMNTqlT3DiMRyn5CxPTQ,15422
|
|
11
|
+
cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
cognite/extractorutils/statestore/__init__.py,sha256=hV3r11FUXkH6-60Ct6zLSROMNVrEeiE3Shmkf28Q-co,359
|
|
13
|
+
cognite/extractorutils/statestore/_base.py,sha256=mWdFk4EZl886V6uXRj4O2sv2_ANJ3Sigmgeql-XEsmc,2675
|
|
14
|
+
cognite/extractorutils/statestore/hashing.py,sha256=Le6PUpLYV7kTKgO2nc5BKCEf-3LTXoGzEVzLtw8tkn0,8011
|
|
15
|
+
cognite/extractorutils/statestore/watermark.py,sha256=U_cA0XlqkgMML-ZeEl13KE8KjQHsId5t7mMHibRhUyA,16713
|
|
16
|
+
cognite/extractorutils/threading.py,sha256=RN9oEXO6N2RqYKThFoDqzSeo593hkzTVePK1KSVOu3A,3586
|
|
17
|
+
cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yjhEYEz9J9WShDfM,341
|
|
18
|
+
cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
cognite/extractorutils/unstable/configuration/exceptions.py,sha256=-cziC11IbUP308ldbAYoQn4x2SNCIxYanN2eIV1n9To,654
|
|
20
|
+
cognite/extractorutils/unstable/configuration/loaders.py,sha256=iMlCx6abKaDHx5-nOQSRtf-creqJPv1QrnbapCaIZkA,3689
|
|
21
|
+
cognite/extractorutils/unstable/configuration/models.py,sha256=jFlA5eEeNRq39KEwAjZV9UkbV2juVUHANNeXq0VtqL4,8210
|
|
22
|
+
cognite/extractorutils/unstable/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
+
cognite/extractorutils/unstable/core/_dto.py,sha256=tvvy39cvf-QT28GWz5FpqxQ5vAVk0t69JoPPhpWlweY,1293
|
|
24
|
+
cognite/extractorutils/unstable/core/_messaging.py,sha256=D9rOW8fijryXffbm90d8VTf2vy5FmwVGU-H0O-cn-EI,68
|
|
25
|
+
cognite/extractorutils/unstable/core/base.py,sha256=QljO7Zpn5RSTEI9PHIavhKWdr4Hp-Ni5tdmsQ_ocOLk,12190
|
|
26
|
+
cognite/extractorutils/unstable/core/errors.py,sha256=D8QAaqwJec62ZbhBNC0flmKjw_EdHLKGn8npqtPQhZE,1706
|
|
27
|
+
cognite/extractorutils/unstable/core/restart_policy.py,sha256=SodG2Gs9Es05yk3EbAAWY_sbSoBUmhTRrUMBR4BSQbQ,622
|
|
28
|
+
cognite/extractorutils/unstable/core/runtime.py,sha256=sb8ouTCZqvzpns_8UpVwPd4nGnfinf7vsVvOk23jksQ,11834
|
|
29
|
+
cognite/extractorutils/unstable/core/tasks.py,sha256=K3R40sNSqYJ1Oc0UMTUDF4lY_WaZ7HokvZ5kctDsjGQ,585
|
|
30
|
+
cognite/extractorutils/unstable/scheduling/__init__.py,sha256=L90_rCZNHvti-PInne0r7W9edIkifctELjiaxEoQiSc,67
|
|
31
|
+
cognite/extractorutils/unstable/scheduling/_scheduler.py,sha256=tzu3-olhBU8uFDYj-Q6mEJUVBVin8wSGJONJVrNP3NE,3694
|
|
32
|
+
cognite/extractorutils/unstable/scheduling/_schedules.py,sha256=y0NVeXYZOFcAyzBgAe8jqK0W-SZL5m99UwXAacGzqIw,677
|
|
33
|
+
cognite/extractorutils/uploader/__init__.py,sha256=MgyvZojwLE-oUCZ0VALISd2rUCqShlyozxhzAKX5uj4,3396
|
|
34
|
+
cognite/extractorutils/uploader/_base.py,sha256=JPr5Dp25XYzwN4MJ2ddd-xhPg5kVV3jASNecD8sAaKs,5273
|
|
35
|
+
cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
|
|
36
|
+
cognite/extractorutils/uploader/assets.py,sha256=SDX48xjqIT4tbQ9HtaIgQT8bw61XHJGic5ofZJeK7UE,5692
|
|
37
|
+
cognite/extractorutils/uploader/data_modeling.py,sha256=Vd9eDWE-KPICChtxcKZdFcH3mSbavD8s1627wXxF_SI,3593
|
|
38
|
+
cognite/extractorutils/uploader/events.py,sha256=qo1rVhk3eUfcbNLauZfvBohQ2aFRazbyGuMFcU-UyQ8,5640
|
|
39
|
+
cognite/extractorutils/uploader/files.py,sha256=7_XtaDS1cxHVQ7tGvib8U5MF0utl-qzXYG4dPYEImTc,25984
|
|
40
|
+
cognite/extractorutils/uploader/raw.py,sha256=VMYfeZN8XAHfZ77AuGcL85bIWvhaO7-Whx_marnGAmQ,6692
|
|
41
|
+
cognite/extractorutils/uploader/time_series.py,sha256=yBN7ppD5hg0CgUIw7WvhhAPyOj0gbIWG4_-ifPaAuOE,26575
|
|
42
|
+
cognite/extractorutils/uploader/upload_failure_handler.py,sha256=Oj3xDK_qlGQdEOzswE-6ti7tDAQXR0Rvee3lg6KBg3s,2000
|
|
43
|
+
cognite/extractorutils/uploader_extractor.py,sha256=X71M_7JcGMwC3kHMETmTF8cdjSQwZaNmIGlT-mBs3Pk,7687
|
|
44
|
+
cognite/extractorutils/uploader_types.py,sha256=eLKFQJT53zpn9_3-SDUtgHUMASGdK7c85HWrLWEF-JE,865
|
|
45
|
+
cognite/extractorutils/util.py,sha256=TL3fkHlvPqWjdyr4yorq5LNJbPxJSom69HKyeQM92xE,21042
|
|
46
|
+
cognite_extractor_utils-7.5.5.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
|
|
47
|
+
cognite_extractor_utils-7.5.5.dist-info/METADATA,sha256=k9onmv9_SPY_5IcXA_QLhuDUSzJyf-RuJMCM1pKn4Tg,5691
|
|
48
|
+
cognite_extractor_utils-7.5.5.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
|
|
49
|
+
cognite_extractor_utils-7.5.5.dist-info/RECORD,,
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Example of how you would build an extractor with the new base class
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
from cognite.extractorutils.unstable.configuration.models import ExtractorConfig
|
|
6
|
-
|
|
7
|
-
from .base import Extractor
|
|
8
|
-
from .runtime import Runtime
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class MyConfig(ExtractorConfig):
|
|
12
|
-
parameter_one: int
|
|
13
|
-
parameter_two: str
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class MyExtractor(Extractor[MyConfig]):
|
|
17
|
-
NAME = "Test extractor"
|
|
18
|
-
EXTERNAL_ID = "test-extractor"
|
|
19
|
-
DESCRIPTION = "Test of the new runtime"
|
|
20
|
-
VERSION = "1.0.0"
|
|
21
|
-
CONFIG_TYPE = MyConfig
|
|
22
|
-
|
|
23
|
-
def run(self) -> None:
|
|
24
|
-
self.logger.info("Started!")
|
|
25
|
-
if not self.cancellation_token.wait(10):
|
|
26
|
-
raise ValueError("Oops")
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if __name__ == "__main__":
|
|
30
|
-
runtime = Runtime(MyExtractor)
|
|
31
|
-
runtime.run()
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
cognite/extractorutils/__init__.py,sha256=3EXu1PpneUoKYLLIAlPg5-z059GC1YWnFCbRNJrDfk0,739
|
|
2
|
-
cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
|
|
3
|
-
cognite/extractorutils/base.py,sha256=q6NU2bPec3WOasVnnIFoh-aUJudVZWZ2R6emz3IRj8Q,16391
|
|
4
|
-
cognite/extractorutils/configtools/__init__.py,sha256=YEpFGJoza23eM8Zj5DqqUj7sEstERV_QYsN6Nw4dKCg,3092
|
|
5
|
-
cognite/extractorutils/configtools/_util.py,sha256=WdZptkZz_vkn1M4Vqwb39Gb1wxTOe_MNJXWHzOtwv50,4797
|
|
6
|
-
cognite/extractorutils/configtools/elements.py,sha256=6HWhh4jVS-v7Wzl9j3D-Gbli7tEtVidnkIT-eQKCOAE,26314
|
|
7
|
-
cognite/extractorutils/configtools/loaders.py,sha256=TEBW9SuvYY-sCq6is0nYSFGM4kcwUdomqcuK9Dv0GbQ,18262
|
|
8
|
-
cognite/extractorutils/configtools/validators.py,sha256=5or-lg8UHNzl7KfZpbVx8b40835RZwHbUQlMsUCAs4w,1053
|
|
9
|
-
cognite/extractorutils/exceptions.py,sha256=1PgvW1FrgVnuNtkwC0RTvG1-FZp1qmBuYrY1AWW-BJc,1188
|
|
10
|
-
cognite/extractorutils/metrics.py,sha256=01ZMRbDisXPxrfCSyTSEkXMsslzmZwEqw18fuu9okdc,15509
|
|
11
|
-
cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
cognite/extractorutils/statestore/__init__.py,sha256=hV3r11FUXkH6-60Ct6zLSROMNVrEeiE3Shmkf28Q-co,359
|
|
13
|
-
cognite/extractorutils/statestore/_base.py,sha256=PM4C-bz41tldA5Lx8rD0AzgXJciAZc2l_1tbz1VV27I,2712
|
|
14
|
-
cognite/extractorutils/statestore/hashing.py,sha256=o-efTv21_ATQnyxYmple3MF7r5Afy-7qZsdZhR47emw,8083
|
|
15
|
-
cognite/extractorutils/statestore/watermark.py,sha256=c_lcmJfo8bOvWyCJ9iRbbE4BlqRVulom4TpHb2pOnkE,16755
|
|
16
|
-
cognite/extractorutils/threading.py,sha256=2Hke5cFvP-wA45Crvh58JahoKXB64P3tr7R4y_BhBqM,3605
|
|
17
|
-
cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yjhEYEz9J9WShDfM,341
|
|
18
|
-
cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
cognite/extractorutils/unstable/configuration/loaders.py,sha256=9TqVLKGiFl7L-6SLucD3zBZLiTm0Va1KkTBne4kpnI8,3350
|
|
20
|
-
cognite/extractorutils/unstable/configuration/models.py,sha256=fPu56TkFKMRr5uc5R3X4WlHNDofvyH0JZfORvwwDT44,7727
|
|
21
|
-
cognite/extractorutils/unstable/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
cognite/extractorutils/unstable/core/__main__.py,sha256=8Tb0RqeRBs47dU5xOikZog-IM2IKVonj9dnPG7Ia710,728
|
|
23
|
-
cognite/extractorutils/unstable/core/_dto.py,sha256=OP_cWZ2EvRbq3Tcczc-NWbYsxPZJ0HPgnKyZw-OAZ-I,1117
|
|
24
|
-
cognite/extractorutils/unstable/core/_messaging.py,sha256=D9rOW8fijryXffbm90d8VTf2vy5FmwVGU-H0O-cn-EI,68
|
|
25
|
-
cognite/extractorutils/unstable/core/base.py,sha256=nvvMc8ddi742M-_CIzsHQSlUswMyn4uAX3ceHp3Db_k,6745
|
|
26
|
-
cognite/extractorutils/unstable/core/runtime.py,sha256=sRrPsICawgEHFU9DYArDpIMxebudR0BZ5K3h6HAW9As,6054
|
|
27
|
-
cognite/extractorutils/unstable/core/tasks.py,sha256=JjzfQDpCmNZxWnSJxmhrK3TvxNbmzRxEjgjHJQQO__c,515
|
|
28
|
-
cognite/extractorutils/unstable/scheduling/__init__.py,sha256=L90_rCZNHvti-PInne0r7W9edIkifctELjiaxEoQiSc,67
|
|
29
|
-
cognite/extractorutils/unstable/scheduling/_scheduler.py,sha256=w2Hs1u3-cNjxrZHtoNFvCmLCd0GNU52K4uUd-Yo_RrM,3691
|
|
30
|
-
cognite/extractorutils/unstable/scheduling/_schedules.py,sha256=y0NVeXYZOFcAyzBgAe8jqK0W-SZL5m99UwXAacGzqIw,677
|
|
31
|
-
cognite/extractorutils/uploader/__init__.py,sha256=W22u6QHA4cR0j78LN5LTL5YGbfC-uTApagTyP5ab7uQ,3110
|
|
32
|
-
cognite/extractorutils/uploader/_base.py,sha256=wktbV8dpb8zBOsNaECZkBNoJSpOz437NlNMER3-a3xQ,5304
|
|
33
|
-
cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
|
|
34
|
-
cognite/extractorutils/uploader/assets.py,sha256=2E90N1kxsaA6Ah4h0_r_dTVhDYY_68ItRWrHYkkltJw,5628
|
|
35
|
-
cognite/extractorutils/uploader/data_modeling.py,sha256=w35Ix5mu0Cgfn4ywnDyif4VVjo04LVTlkMEevk6ztUs,3639
|
|
36
|
-
cognite/extractorutils/uploader/events.py,sha256=NZP2tMoU_rh_rb-EZiUBsOT5KdNABHN4c9Oddk0OsdE,5680
|
|
37
|
-
cognite/extractorutils/uploader/files.py,sha256=OYBLQXzS1HLDd_g5KrcwmwITjCVYLqXKuP0lWFrgpr0,23047
|
|
38
|
-
cognite/extractorutils/uploader/raw.py,sha256=wFjF90PFTjmByOWx_Y4_YfDJ2w2jl0EQJ2Tjx2MP2PM,6738
|
|
39
|
-
cognite/extractorutils/uploader/time_series.py,sha256=HBtQdsQoIOaL-EG5lMsaY-ORwVb0kGiXG86VjE5-_Bg,26815
|
|
40
|
-
cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
|
|
41
|
-
cognite/extractorutils/uploader_types.py,sha256=wxfrsiKPTzG5lmoYtQsxt8Xyj-s5HnaLl8WDzJNrazg,1020
|
|
42
|
-
cognite/extractorutils/util.py,sha256=cFrbO8XCFT7QBs2vvOk4dWyOvaHclvsuPlrDJYFG0xA,21160
|
|
43
|
-
cognite_extractor_utils-7.5.4.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
|
|
44
|
-
cognite_extractor_utils-7.5.4.dist-info/METADATA,sha256=ylZnUNTFglZUVhZQsiVIOg-ChKYpKO6lsfhIR2g8JsM,5649
|
|
45
|
-
cognite_extractor_utils-7.5.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
46
|
-
cognite_extractor_utils-7.5.4.dist-info/RECORD,,
|
|
File without changes
|