splight-lib 5.11.5.dev0__tar.gz → 5.11.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/PKG-INFO +1 -1
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/pyproject.toml +1 -1
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/auth/__init__.py +0 -2
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/database/abstract.py +3 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/database/builder.py +2 -2
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/database/remote_client.py +23 -23
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/buffer.py +3 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/builder.py +2 -2
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/hub/abstract.py +3 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/hub/client.py +10 -10
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/abstract.py +6 -6
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/spec.py +27 -27
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/encryption.py +1 -2
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/engine.py +2 -3
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/scheduling.py +10 -11
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/task.py +6 -6
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/trigger.py +3 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/__init__.py +28 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/alert.py +34 -32
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/asset.py +12 -13
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/attribute.py +3 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/component.py +56 -58
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/dashboard.py +77 -38
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/data_address.py +2 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/datalake.py +34 -6
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/file.py +8 -9
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/function.py +17 -16
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/hub.py +28 -27
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/hub_solution.py +6 -7
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/metadata.py +5 -5
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/secret.py +3 -3
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/server.py +1 -1
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/solution.py +3 -3
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tag.py +1 -3
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/settings.py +5 -5
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/stringcase.py +8 -7
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/testing/__init__.py +4 -4
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/utils/custom_model.py +7 -7
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/utils/hub.py +3 -3
- splight_lib-5.11.5.dev0/splight_lib/auth/exceptions.py +0 -12
- splight_lib-5.11.5.dev0/splight_lib/auth/mac_auth.py +0 -96
- splight_lib-5.11.5.dev0/splight_lib/client/file_handler.py +0 -34
- splight_lib-5.11.5.dev0/splight_lib/client/filter.py +0 -18
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/LICENSE.txt +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/README.md +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/abstract/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/abstract/client.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/auth/token.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/database/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/database/classmap.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/abstract.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/constants.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/datalake/remote_client.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/hub/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/tests/test_database.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/client/tests/test_datalake.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/tests/test_abstract.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/component/tests/test_spec.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/conftest.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/constants.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/tests/test_execution.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/execution/tests/test_scheduling.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/_internal.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/component.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/constants.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/logging.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/logging/tests/test_logging.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/actions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/database_base.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/datalake_base.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/generic.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/hub_server.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/native.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tests/models.json +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tests/test_component_object_instance.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tests/test_database_model.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tests/test_metadata.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/tests/test_models.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/models/variable_types.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/restclient/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/restclient/client.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/restclient/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/restclient/tests/test_restclient.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/restclient/types.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/server/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/server/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/server/server.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/solution/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/solution/exceptions.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/solution/solution.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/tests/FakeProc.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/tests/asset_geometries.json +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/tests/test_api_contracts.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/tests/test_encryption.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/utils/__init__.py +0 -0
- {splight_lib-5.11.5.dev0 → splight_lib-5.11.7}/splight_lib/version.py +0 -0
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
from abc import abstractmethod
|
|
2
2
|
from tempfile import NamedTemporaryFile
|
|
3
|
-
from typing import Dict, List, Union
|
|
4
3
|
|
|
5
4
|
from splight_lib.abstract.client import AbstractClient, QuerySet
|
|
6
5
|
|
|
7
6
|
|
|
8
7
|
class AbstractDatabaseClient(AbstractClient):
|
|
9
8
|
@abstractmethod
|
|
10
|
-
def save(self, resource_name: str, instance:
|
|
9
|
+
def save(self, resource_name: str, instance: dict) -> dict:
|
|
11
10
|
pass
|
|
12
11
|
|
|
13
12
|
@abstractmethod
|
|
14
13
|
def _get(
|
|
15
14
|
self, resource_name: str, first: bool = False, **kwargs
|
|
16
|
-
) ->
|
|
15
|
+
) -> dict | list[dict]:
|
|
17
16
|
pass
|
|
18
17
|
|
|
19
18
|
def get(self, resource_name: str, *args, **kwargs) -> QuerySet:
|
|
@@ -24,5 +23,5 @@ class AbstractDatabaseClient(AbstractClient):
|
|
|
24
23
|
pass
|
|
25
24
|
|
|
26
25
|
@abstractmethod
|
|
27
|
-
def download(self, instance:
|
|
26
|
+
def download(self, instance: dict) -> NamedTemporaryFile:
|
|
28
27
|
pass
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
3
|
from splight_lib.client.database.abstract import AbstractDatabaseClient
|
|
4
4
|
from splight_lib.client.database.remote_client import RemoteDatabaseClient
|
|
@@ -6,6 +6,6 @@ from splight_lib.client.database.remote_client import RemoteDatabaseClient
|
|
|
6
6
|
|
|
7
7
|
class DatabaseClientBuilder:
|
|
8
8
|
@staticmethod
|
|
9
|
-
def build(parameters:
|
|
9
|
+
def build(parameters: dict[str, Any] = {}) -> AbstractDatabaseClient:
|
|
10
10
|
db_client = RemoteDatabaseClient(**parameters)
|
|
11
11
|
return db_client
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from tempfile import NamedTemporaryFile
|
|
2
|
-
from typing import Any,
|
|
2
|
+
from typing import Any, Generator
|
|
3
3
|
|
|
4
4
|
import progressbar
|
|
5
5
|
import requests
|
|
@@ -31,9 +31,9 @@ logger = get_splight_logger()
|
|
|
31
31
|
|
|
32
32
|
class PaginatedResponse(TypedDict):
|
|
33
33
|
count: int
|
|
34
|
-
next:
|
|
35
|
-
previous:
|
|
36
|
-
results:
|
|
34
|
+
next: str | None
|
|
35
|
+
previous: str | None
|
|
36
|
+
results: list[Any]
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
@@ -66,9 +66,9 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
66
66
|
def save(
|
|
67
67
|
self,
|
|
68
68
|
resource_name: str,
|
|
69
|
-
instance:
|
|
70
|
-
files:
|
|
71
|
-
) ->
|
|
69
|
+
instance: dict,
|
|
70
|
+
files: dict[str, str] | None = None,
|
|
71
|
+
) -> dict:
|
|
72
72
|
"""Creates or updates a resource depending on the name if
|
|
73
73
|
it contains the id or not.
|
|
74
74
|
|
|
@@ -126,7 +126,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
126
126
|
resource_name: str,
|
|
127
127
|
first: bool = False,
|
|
128
128
|
**kwargs,
|
|
129
|
-
) ->
|
|
129
|
+
) -> dict | list[dict]:
|
|
130
130
|
"""Retrieves one or multiple resources. If the parameter id is passed
|
|
131
131
|
as a kwarg, the instance with that id will be retrieved.
|
|
132
132
|
|
|
@@ -149,7 +149,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
149
149
|
)
|
|
150
150
|
return instances
|
|
151
151
|
|
|
152
|
-
def operate(self, resource_name: str, instance:
|
|
152
|
+
def operate(self, resource_name: str, instance: dict) -> dict:
|
|
153
153
|
model_name = resource_name.lower()
|
|
154
154
|
api_path = CUSTOM_PATHS_MAP.get(model_name)
|
|
155
155
|
if not api_path:
|
|
@@ -163,7 +163,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
163
163
|
|
|
164
164
|
def _retrieve_multiple(
|
|
165
165
|
self, resource_name: str, first: bool = False, **kwargs
|
|
166
|
-
) ->
|
|
166
|
+
) -> list[dict]:
|
|
167
167
|
logger.debug(f"Retrieving objects {resource_name}")
|
|
168
168
|
api_path = self._get_api_path(resource_name)
|
|
169
169
|
url = self._base_url / api_path
|
|
@@ -176,7 +176,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
176
176
|
return instances
|
|
177
177
|
|
|
178
178
|
@retry(SPLIGHT_REQUEST_EXCEPTIONS, tries=3, delay=1)
|
|
179
|
-
def _retrieve_single(self, resource_name: str, id: str) ->
|
|
179
|
+
def _retrieve_single(self, resource_name: str, id: str) -> dict:
|
|
180
180
|
logger.debug(f"Retrieving object {resource_name} with id {id}")
|
|
181
181
|
api_path = self._get_api_path(resource_name)
|
|
182
182
|
url = self._base_url / api_path / f"{id}/"
|
|
@@ -191,8 +191,8 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
191
191
|
def download(
|
|
192
192
|
self,
|
|
193
193
|
resource_name: str,
|
|
194
|
-
instance:
|
|
195
|
-
type_:
|
|
194
|
+
instance: dict,
|
|
195
|
+
type_: str | None = None,
|
|
196
196
|
**kwargs,
|
|
197
197
|
) -> NamedTemporaryFile:
|
|
198
198
|
"""Returns the number of resources in the database for a given model
|
|
@@ -271,9 +271,9 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
271
271
|
def _create(
|
|
272
272
|
self,
|
|
273
273
|
resource_name: str,
|
|
274
|
-
instance:
|
|
275
|
-
files:
|
|
276
|
-
) ->
|
|
274
|
+
instance: dict,
|
|
275
|
+
files: dict[str, str] | None = None,
|
|
276
|
+
) -> dict:
|
|
277
277
|
logger.debug("Saving new instance", tags=LogTags.DATABASE)
|
|
278
278
|
model_name = resource_name.lower()
|
|
279
279
|
api_path = self._get_api_path(resource_name)
|
|
@@ -305,9 +305,9 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
305
305
|
self,
|
|
306
306
|
resource_name: str,
|
|
307
307
|
resource_id: str,
|
|
308
|
-
instance:
|
|
309
|
-
files:
|
|
310
|
-
) ->
|
|
308
|
+
instance: dict,
|
|
309
|
+
files: dict[str, str] | None = None,
|
|
310
|
+
) -> dict:
|
|
311
311
|
logger.debug("Saving instance %s", resource_id, tags=LogTags.DATABASE)
|
|
312
312
|
model_name = resource_name.lower()
|
|
313
313
|
api_path = self._get_api_path(resource_name)
|
|
@@ -323,7 +323,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
323
323
|
raise RequestError(response.status_code, response.text)
|
|
324
324
|
return response.json()
|
|
325
325
|
|
|
326
|
-
def _create_file(self, instance:
|
|
326
|
+
def _create_file(self, instance: dict, url: furl):
|
|
327
327
|
response = self._restclient.post(url, data=instance)
|
|
328
328
|
if response.is_error:
|
|
329
329
|
raise RequestError(response.status_code, response.text)
|
|
@@ -333,7 +333,7 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
333
333
|
self._upload_file(created_instance, file_path=file_path)
|
|
334
334
|
return created_instance
|
|
335
335
|
|
|
336
|
-
def _upload_file(self, instance:
|
|
336
|
+
def _upload_file(self, instance: dict, file_path: str):
|
|
337
337
|
api_path = self._get_api_path("file")
|
|
338
338
|
resource_id = instance.get("id")
|
|
339
339
|
url = self._base_url / api_path / f"{resource_id}/upload_url/"
|
|
@@ -358,9 +358,9 @@ class RemoteDatabaseClient(AbstractDatabaseClient, AbstractRemoteClient):
|
|
|
358
358
|
def upload(
|
|
359
359
|
self,
|
|
360
360
|
resource_name: str,
|
|
361
|
-
instance:
|
|
361
|
+
instance: dict,
|
|
362
362
|
file_path: str,
|
|
363
|
-
type_:
|
|
363
|
+
type_: str | None = None,
|
|
364
364
|
):
|
|
365
365
|
api_path = self._get_api_path(resource_name)
|
|
366
366
|
resource_id = instance.get("id")
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from datetime import datetime, timezone
|
|
2
|
-
from typing import Dict, List
|
|
3
2
|
|
|
4
3
|
|
|
5
4
|
class DatalakeDocumentBuffer:
|
|
@@ -8,13 +7,13 @@ class DatalakeDocumentBuffer:
|
|
|
8
7
|
def __init__(self, buffer_size: int, buffer_timeout: float):
|
|
9
8
|
self._size = buffer_size
|
|
10
9
|
self._timeout = buffer_timeout
|
|
11
|
-
self._buffer:
|
|
10
|
+
self._buffer: list[dict] = []
|
|
12
11
|
self._last_flush = datetime.now(timezone.utc)
|
|
13
12
|
|
|
14
13
|
self.reset()
|
|
15
14
|
|
|
16
15
|
@property
|
|
17
|
-
def data(self) ->
|
|
16
|
+
def data(self) -> list[dict]:
|
|
18
17
|
"""Retrieves the buffer data
|
|
19
18
|
|
|
20
19
|
Returns
|
|
@@ -45,7 +44,7 @@ class DatalakeDocumentBuffer:
|
|
|
45
44
|
self._last_flush = datetime.now(timezone.utc)
|
|
46
45
|
self._buffer = []
|
|
47
46
|
|
|
48
|
-
def add_documents(self, documents:
|
|
47
|
+
def add_documents(self, documents: list[dict]):
|
|
49
48
|
"""Adds new documents to the buffer.
|
|
50
49
|
|
|
51
50
|
Parameters
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
3
|
from splight_lib.client.datalake.abstract import AbstractDatalakeClient
|
|
4
4
|
from splight_lib.client.datalake.remote_client import (
|
|
@@ -19,6 +19,6 @@ class DatalakeClientBuilder:
|
|
|
19
19
|
@staticmethod
|
|
20
20
|
def build(
|
|
21
21
|
dl_client_type: DatalakeClientType = DatalakeClientType.BUFFERED_ASYNC,
|
|
22
|
-
parameters:
|
|
22
|
+
parameters: dict[str, Any] = {},
|
|
23
23
|
) -> AbstractDatalakeClient:
|
|
24
24
|
return DL_CLIENT_TYPE_MAP[dl_client_type](**parameters)
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from abc import abstractmethod
|
|
2
|
-
from typing import Dict, List, Tuple
|
|
3
2
|
|
|
4
3
|
from pydantic import BaseModel
|
|
5
4
|
|
|
@@ -17,7 +16,7 @@ class AbstractHubClient(AbstractClient):
|
|
|
17
16
|
limit_: int = -1,
|
|
18
17
|
skip_: int = 0,
|
|
19
18
|
**kwargs,
|
|
20
|
-
) ->
|
|
19
|
+
) -> list[BaseModel]:
|
|
21
20
|
pass
|
|
22
21
|
|
|
23
22
|
@abstractmethod
|
|
@@ -25,7 +24,7 @@ class AbstractHubClient(AbstractClient):
|
|
|
25
24
|
pass
|
|
26
25
|
|
|
27
26
|
@abstractmethod
|
|
28
|
-
def download(self, data:
|
|
27
|
+
def download(self, data: dict) -> tuple:
|
|
29
28
|
pass
|
|
30
29
|
|
|
31
30
|
@abstractmethod
|
|
@@ -33,5 +32,5 @@ class AbstractHubClient(AbstractClient):
|
|
|
33
32
|
pass
|
|
34
33
|
|
|
35
34
|
@abstractmethod
|
|
36
|
-
def save(self, instance:
|
|
35
|
+
def save(self, instance: dict) -> dict:
|
|
37
36
|
pass
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from tempfile import NamedTemporaryFile
|
|
2
|
-
from typing import Any,
|
|
2
|
+
from typing import Any, Generator, TypedDict
|
|
3
3
|
|
|
4
4
|
import progressbar
|
|
5
5
|
import requests
|
|
@@ -13,9 +13,9 @@ from splight_lib.client.hub.abstract import AbstractHubClient
|
|
|
13
13
|
|
|
14
14
|
class PaginatedResponse(TypedDict):
|
|
15
15
|
count: int
|
|
16
|
-
next:
|
|
17
|
-
previous:
|
|
18
|
-
results:
|
|
16
|
+
next: str | None
|
|
17
|
+
previous: str | None
|
|
18
|
+
results: list[Any]
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class SplightHubClient(AbstractHubClient):
|
|
@@ -58,7 +58,7 @@ class SplightHubClient(AbstractHubClient):
|
|
|
58
58
|
limit_: int = -1,
|
|
59
59
|
skip_: int = 0,
|
|
60
60
|
**kwargs,
|
|
61
|
-
) ->
|
|
61
|
+
) -> list[BaseModel]:
|
|
62
62
|
url = self._hub_url / "versions/"
|
|
63
63
|
params = self._get_params(limit_, skip_, **kwargs)
|
|
64
64
|
instances = []
|
|
@@ -71,14 +71,14 @@ class SplightHubClient(AbstractHubClient):
|
|
|
71
71
|
def get_org_id(self):
|
|
72
72
|
return self._org_id
|
|
73
73
|
|
|
74
|
-
def _create(self, instance:
|
|
74
|
+
def _create(self, instance: dict) -> dict:
|
|
75
75
|
url = self._hub_url / "components/"
|
|
76
76
|
response = self._session.post(url, json=instance)
|
|
77
77
|
response.raise_for_status()
|
|
78
78
|
instance = response.json()
|
|
79
79
|
return instance
|
|
80
80
|
|
|
81
|
-
def _update(self, instance:
|
|
81
|
+
def _update(self, instance: dict) -> dict:
|
|
82
82
|
instance_id = instance.get("id")
|
|
83
83
|
url = self._hub_url / "versions" / f"{instance_id}/"
|
|
84
84
|
response = self._session.put(url, json=instance)
|
|
@@ -86,12 +86,12 @@ class SplightHubClient(AbstractHubClient):
|
|
|
86
86
|
instance = response.json()
|
|
87
87
|
return instance
|
|
88
88
|
|
|
89
|
-
def build(self, id: str):
|
|
89
|
+
def build(self, id: str) -> None:
|
|
90
90
|
url = self._hub_url / f"versions/{id}/build/"
|
|
91
91
|
response = self._session.post(url)
|
|
92
92
|
response.raise_for_status()
|
|
93
93
|
|
|
94
|
-
def upload(self, id: str, file_path: str, type_: str):
|
|
94
|
+
def upload(self, id: str, file_path: str, type_: str) -> None:
|
|
95
95
|
url = self._hub_url / f"versions/{id}/upload_url/"
|
|
96
96
|
params = {"type": type_}
|
|
97
97
|
response = self._session.get(url, params=params)
|
|
@@ -135,7 +135,7 @@ class SplightHubClient(AbstractHubClient):
|
|
|
135
135
|
response.status_code == 204
|
|
136
136
|
), f"Failed to delete component: {response.json()}"
|
|
137
137
|
|
|
138
|
-
def save(self, instance:
|
|
138
|
+
def save(self, instance: dict) -> dict:
|
|
139
139
|
if instance.get("id"):
|
|
140
140
|
return self._update(instance)
|
|
141
141
|
else:
|
|
@@ -5,7 +5,7 @@ from collections import namedtuple
|
|
|
5
5
|
from tempfile import NamedTemporaryFile
|
|
6
6
|
from threading import Thread
|
|
7
7
|
from time import sleep
|
|
8
|
-
from typing import Callable,
|
|
8
|
+
from typing import Callable, Type
|
|
9
9
|
|
|
10
10
|
from pydantic import BaseModel
|
|
11
11
|
from pydantic_core import ValidationError
|
|
@@ -70,7 +70,7 @@ class HealthCheckProcessor:
|
|
|
70
70
|
class SplightBaseComponent(ABC):
|
|
71
71
|
def __init__(
|
|
72
72
|
self,
|
|
73
|
-
component_id:
|
|
73
|
+
component_id: str | None = None,
|
|
74
74
|
):
|
|
75
75
|
self._component_id = component_id
|
|
76
76
|
self._execution_engine = ExecutionEngine()
|
|
@@ -185,7 +185,7 @@ class SplightBaseComponent(ABC):
|
|
|
185
185
|
return wrapper
|
|
186
186
|
|
|
187
187
|
def _get_custom_type_model(
|
|
188
|
-
self, component_object:
|
|
188
|
+
self, component_object: dict[str, Type[ComponentObjectInstance]]
|
|
189
189
|
) -> BaseModel:
|
|
190
190
|
custom_type_model = namedtuple(
|
|
191
191
|
"CustomTypes", [k for k in component_object.keys()]
|
|
@@ -193,7 +193,7 @@ class SplightBaseComponent(ABC):
|
|
|
193
193
|
return custom_type_model(**component_object)
|
|
194
194
|
|
|
195
195
|
def _get_routine_model(
|
|
196
|
-
self, routine_objects:
|
|
196
|
+
self, routine_objects: dict[str, Type[RoutineObjectInstance]]
|
|
197
197
|
) -> namedtuple:
|
|
198
198
|
routine_model = namedtuple(
|
|
199
199
|
"ComponentRoutine", [k for k in routine_objects.keys()]
|
|
@@ -209,9 +209,9 @@ class SplightBaseComponent(ABC):
|
|
|
209
209
|
return spec
|
|
210
210
|
|
|
211
211
|
@abstractmethod
|
|
212
|
-
def start(self):
|
|
212
|
+
def start(self) -> None:
|
|
213
213
|
raise NotImplementedError()
|
|
214
214
|
|
|
215
|
-
def stop(self):
|
|
215
|
+
def stop(self) -> None:
|
|
216
216
|
self._execution_engine.stop()
|
|
217
217
|
sys.exit(1)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import json
|
|
2
2
|
from collections import namedtuple
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Annotated, Type
|
|
4
4
|
|
|
5
5
|
from pydantic import AnyUrl, BaseModel, Field, ValidationInfo, field_validator
|
|
6
6
|
|
|
@@ -43,7 +43,7 @@ VALID_DEPENDS_ON = [
|
|
|
43
43
|
]
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def check_unique_values(values:
|
|
46
|
+
def check_unique_values(values: list[str]):
|
|
47
47
|
"""Checks if there are repeated values in a list of strings.
|
|
48
48
|
|
|
49
49
|
Raises
|
|
@@ -54,14 +54,14 @@ def check_unique_values(values: List[str]):
|
|
|
54
54
|
raise DuplicatedValuesError("The list contains duplicated values")
|
|
55
55
|
|
|
56
56
|
|
|
57
|
-
def check_parameter_dependency(parameters:
|
|
57
|
+
def check_parameter_dependency(parameters: list[InputParameter]):
|
|
58
58
|
"""Checks dependency between parameters.
|
|
59
59
|
|
|
60
60
|
Raises
|
|
61
61
|
------
|
|
62
62
|
ParameterDependencyError thrown for an error in the dependency.
|
|
63
63
|
"""
|
|
64
|
-
parameter_map:
|
|
64
|
+
parameter_map: dict[str, InputParameter] = {p.name: p for p in parameters}
|
|
65
65
|
for parameter in parameters:
|
|
66
66
|
if not parameter.depends_on:
|
|
67
67
|
continue
|
|
@@ -75,35 +75,35 @@ def check_parameter_dependency(parameters: List[InputParameter]):
|
|
|
75
75
|
|
|
76
76
|
|
|
77
77
|
class Spec(BaseModel):
|
|
78
|
-
name: str
|
|
79
|
-
version: str
|
|
80
|
-
splight_lib_version:
|
|
81
|
-
None, pattern=r"^(\d+)\.(\d+)\.(\d+)(\.dev[0-9]+)?$"
|
|
82
|
-
|
|
83
|
-
splight_cli_version:
|
|
84
|
-
None, pattern=r"^(\d+)\.(\d+)\.(\d+)(\.dev[0-9]+)?$"
|
|
85
|
-
|
|
78
|
+
name: Annotated[str, Field(pattern=r"^[a-zA-Z0-9\s]+$")]
|
|
79
|
+
version: Annotated[str, Field(pattern=r"^(\d+\.)?(\d+\.)?(\*|\d+)$")]
|
|
80
|
+
splight_lib_version: Annotated[
|
|
81
|
+
str | None, Field(pattern=r"^(\d+)\.(\d+)\.(\d+)(\.dev[0-9]+)?$")
|
|
82
|
+
] = None
|
|
83
|
+
splight_cli_version: Annotated[
|
|
84
|
+
str | None, Field(pattern=r"^(\d+)\.(\d+)\.(\d+)(\.dev[0-9]+)?$")
|
|
85
|
+
] = None
|
|
86
86
|
description: str | None = None
|
|
87
87
|
privacy_policy: PrivacyPolicy = PrivacyPolicy.PUBLIC
|
|
88
|
-
tags:
|
|
88
|
+
tags: set[str] = set()
|
|
89
89
|
component_type: ComponentType = ComponentType.CONNECTOR
|
|
90
|
-
custom_types:
|
|
91
|
-
input:
|
|
92
|
-
output:
|
|
93
|
-
routines:
|
|
94
|
-
endpoints:
|
|
90
|
+
custom_types: list[CustomType] = []
|
|
91
|
+
input: list[InputParameter] = []
|
|
92
|
+
output: list[Output] = []
|
|
93
|
+
routines: list[Routine] = []
|
|
94
|
+
endpoints: list[Endpoint] = []
|
|
95
95
|
|
|
96
96
|
@field_validator("custom_types", mode="after")
|
|
97
97
|
def validate_custom_types(
|
|
98
|
-
cls, custom_types:
|
|
99
|
-
) ->
|
|
98
|
+
cls, custom_types: list[CustomType], info: ValidationInfo
|
|
99
|
+
) -> list[CustomType]:
|
|
100
100
|
try:
|
|
101
101
|
check_unique_values([item.name for item in custom_types])
|
|
102
102
|
except DuplicatedValuesError as exc:
|
|
103
103
|
raise ValueError("Repeated Custom Types in spec") from exc
|
|
104
104
|
|
|
105
|
-
valid_types:
|
|
106
|
-
custom_type_names:
|
|
105
|
+
valid_types: list[str] = VALID_PARAMETER_VALUES.keys()
|
|
106
|
+
custom_type_names: list[str] = []
|
|
107
107
|
|
|
108
108
|
for custom_type in custom_types:
|
|
109
109
|
for field in custom_type.fields:
|
|
@@ -125,16 +125,16 @@ class Spec(BaseModel):
|
|
|
125
125
|
|
|
126
126
|
@field_validator("input", mode="after")
|
|
127
127
|
def validate_parameters(
|
|
128
|
-
cls, input:
|
|
129
|
-
) ->
|
|
128
|
+
cls, input: list[InputParameter], info: ValidationInfo
|
|
129
|
+
) -> list[InputParameter]:
|
|
130
130
|
values = info.data
|
|
131
131
|
try:
|
|
132
132
|
check_unique_values([item.name for item in input])
|
|
133
133
|
except DuplicatedValuesError as exc:
|
|
134
134
|
raise ValueError("Repeated Input parameters in spec") from exc
|
|
135
135
|
|
|
136
|
-
custom_type_names:
|
|
137
|
-
valid_types_names:
|
|
136
|
+
custom_type_names: list[str] = [c.name for c in values["custom_types"]]
|
|
137
|
+
valid_types_names: list[str] = (
|
|
138
138
|
list(VALID_PARAMETER_VALUES.keys()) + custom_type_names
|
|
139
139
|
)
|
|
140
140
|
|
|
@@ -149,7 +149,7 @@ class Spec(BaseModel):
|
|
|
149
149
|
return input
|
|
150
150
|
|
|
151
151
|
@field_validator("output", mode="after")
|
|
152
|
-
def validate_output(cls, output:
|
|
152
|
+
def validate_output(cls, output: list[Output]) -> list[Output]:
|
|
153
153
|
try:
|
|
154
154
|
check_unique_values([item.name for item in output])
|
|
155
155
|
except DuplicatedValuesError as exc:
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from functools import cached_property
|
|
2
|
-
from typing import Optional
|
|
3
2
|
|
|
4
3
|
from cryptography.fernet import Fernet as Branca
|
|
5
4
|
from pydantic_settings import BaseSettings
|
|
@@ -10,7 +9,7 @@ logger = get_splight_logger()
|
|
|
10
9
|
|
|
11
10
|
|
|
12
11
|
class EncryptionSettings(BaseSettings):
|
|
13
|
-
SPLIGHT_ENCRYPTION_KEY:
|
|
12
|
+
SPLIGHT_ENCRYPTION_KEY: str | None = None
|
|
14
13
|
|
|
15
14
|
|
|
16
15
|
class EncryptionClient:
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from enum import auto
|
|
2
|
-
from typing import Set, Tuple
|
|
3
2
|
|
|
4
3
|
import pytz
|
|
5
4
|
from apscheduler.events import EVENT_JOB_ERROR, JobExecutionEvent
|
|
@@ -35,7 +34,7 @@ class ExecutionEngine:
|
|
|
35
34
|
timezone=pytz.UTC,
|
|
36
35
|
daemon=True,
|
|
37
36
|
)
|
|
38
|
-
self._critical_jobs:
|
|
37
|
+
self._critical_jobs: set[str] = set()
|
|
39
38
|
self._blocking_sch.add_listener(
|
|
40
39
|
self._task_fail_callback, EVENT_JOB_ERROR
|
|
41
40
|
)
|
|
@@ -53,7 +52,7 @@ class ExecutionEngine:
|
|
|
53
52
|
def state(self) -> EngineStatus:
|
|
54
53
|
return self._state
|
|
55
54
|
|
|
56
|
-
def healthcheck(self) ->
|
|
55
|
+
def healthcheck(self) -> tuple[bool, str]:
|
|
57
56
|
return (self._running, self._state.value)
|
|
58
57
|
|
|
59
58
|
def start(self):
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from datetime import datetime, timezone
|
|
2
|
-
from typing import Optional, Union
|
|
3
2
|
|
|
4
3
|
from pydantic import (
|
|
5
4
|
BaseModel,
|
|
@@ -22,7 +21,7 @@ FIELD_RANGE = {
|
|
|
22
21
|
|
|
23
22
|
|
|
24
23
|
def validate_value_in_range(
|
|
25
|
-
value:
|
|
24
|
+
value: int | str, min_value: int, max_value: int, name: str
|
|
26
25
|
):
|
|
27
26
|
if isinstance(value, str):
|
|
28
27
|
if "*" in value:
|
|
@@ -48,18 +47,18 @@ class TaskPeriod(BaseModel):
|
|
|
48
47
|
minutes: int = 0
|
|
49
48
|
seconds: int = 0
|
|
50
49
|
start_date: datetime = datetime.now(timezone.utc)
|
|
51
|
-
end_date:
|
|
50
|
+
end_date: datetime | None = None
|
|
52
51
|
|
|
53
52
|
|
|
54
53
|
class Crontab(BaseModel):
|
|
55
|
-
year:
|
|
56
|
-
month:
|
|
57
|
-
day:
|
|
58
|
-
week:
|
|
59
|
-
day_of_week:
|
|
60
|
-
hour:
|
|
61
|
-
minute:
|
|
62
|
-
second:
|
|
54
|
+
year: str | int | None = None
|
|
55
|
+
month: str | int | None = None
|
|
56
|
+
day: str | int | None = None
|
|
57
|
+
week: str | int | None = None
|
|
58
|
+
day_of_week: str | int | None = None
|
|
59
|
+
hour: str | int | None = None
|
|
60
|
+
minute: str | int | None = None
|
|
61
|
+
second: str | int | None = None
|
|
63
62
|
|
|
64
63
|
@field_validator(
|
|
65
64
|
"month", "day", "week", "day_of_week", "hour", "minute", "second"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
-
from typing import Callable
|
|
2
|
+
from typing import Callable
|
|
3
3
|
|
|
4
4
|
import pytz
|
|
5
5
|
from apscheduler.triggers.cron import CronTrigger
|
|
@@ -10,7 +10,7 @@ from splight_lib.execution.trigger import IntervalTrigger
|
|
|
10
10
|
|
|
11
11
|
class BaseTask(ABC):
|
|
12
12
|
@abstractmethod
|
|
13
|
-
def as_job(self) ->
|
|
13
|
+
def as_job(self) -> dict:
|
|
14
14
|
pass
|
|
15
15
|
|
|
16
16
|
|
|
@@ -18,8 +18,8 @@ class PeriodicTask(BaseTask):
|
|
|
18
18
|
def __init__(
|
|
19
19
|
self,
|
|
20
20
|
target: Callable,
|
|
21
|
-
period:
|
|
22
|
-
target_args:
|
|
21
|
+
period: TaskPeriod | int,
|
|
22
|
+
target_args: tuple | None = None,
|
|
23
23
|
):
|
|
24
24
|
self._target = target
|
|
25
25
|
self._args = target_args
|
|
@@ -50,7 +50,7 @@ class CronnedTask(BaseTask):
|
|
|
50
50
|
self,
|
|
51
51
|
target: Callable,
|
|
52
52
|
crontab: Crontab,
|
|
53
|
-
target_args:
|
|
53
|
+
target_args: tuple | None = None,
|
|
54
54
|
):
|
|
55
55
|
self._target = target
|
|
56
56
|
self._trigger = CronTrigger(**dict(crontab), timezone=pytz.UTC)
|
|
@@ -61,7 +61,7 @@ class CronnedTask(BaseTask):
|
|
|
61
61
|
cls,
|
|
62
62
|
target: Callable,
|
|
63
63
|
cron_str: str,
|
|
64
|
-
target_args:
|
|
64
|
+
target_args: tuple | None = None,
|
|
65
65
|
):
|
|
66
66
|
crontab = Crontab.from_string(cron_str)
|
|
67
67
|
return cls(target, crontab, target_args)
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from datetime import datetime, tzinfo
|
|
2
|
-
from typing import Optional, Union
|
|
3
2
|
|
|
4
3
|
from apscheduler.triggers.interval import (
|
|
5
4
|
IntervalTrigger as BaseIntervalTrigger,
|
|
@@ -17,9 +16,9 @@ class IntervalTrigger(BaseIntervalTrigger):
|
|
|
17
16
|
hours: int = 0,
|
|
18
17
|
minutes: int = 0,
|
|
19
18
|
seconds: int = 0,
|
|
20
|
-
start_date:
|
|
21
|
-
end_date:
|
|
22
|
-
timezone:
|
|
19
|
+
start_date: datetime | str | None = None,
|
|
20
|
+
end_date: datetime | str | None = None,
|
|
21
|
+
timezone: tzinfo | str | None = None,
|
|
23
22
|
jitter: int = None,
|
|
24
23
|
run_on_start: bool = True,
|
|
25
24
|
):
|