splight-lib 5.9.6.dev2__tar.gz → 5.10.1.dev0__tar.gz

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-lib
-Version: 5.9.6.dev2
+Version: 5.10.1.dev0
 Summary: Splight Library
 Author: Splight Dev
 Author-email: dev@splight-ae.com

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "splight-lib"
-version = "5.9.6.dev2"
+version = "5.10.1dev0"
 description = "Splight Library"
 authors = ["Splight Dev <dev@splight-ae.com>"]
 readme = "README.md"
@@ -71,4 +71,52 @@ filterwarnings = [
 ]
 
 [tool.ruff]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
 line-length = 79
+indent-width = 4
+
+# Assume Python 3.8
+target-version = "py38"
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/client/database/classmap.py

@@ -36,4 +36,5 @@ CUSTOM_PATHS_MAP = {
     "get-asset-attribute": "{prefix}/assets/{asset}/get-attribute/",
     "decrypt-secret": "{prefix}/secrets/decrypt/",
     "routine-status": "{prefix}/component/routines/{routine}/update_status/",
+    "server-status": "{prefix}/server/servers/{server}/update-status/",
 }

splight_lib-5.10.1.dev0/splight_lib/client/datalake/abstract.py

@@ -0,0 +1,36 @@
+from abc import abstractmethod
+from typing import Any, TypedDict
+
+from splight_lib.abstract.client import AbstractRemoteClient, QuerySet
+
+
+class Records(TypedDict):
+    collection: str
+    records: list[dict[str, Any]]
+
+
+class AbstractDatalakeClient(AbstractRemoteClient):
+    def get(self, *args, **kwargs) -> QuerySet:
+        kwargs["get_func"] = "_get"
+        kwargs["count_func"] = "None"
+        return QuerySet(self, *args, **kwargs)
+
+    async def async_get(self, *args, **kwargs):
+        # TODO: consider using an async QuerySet
+        return await self._async_get(*args, **kwargs)
+
+    @abstractmethod
+    def save(self, records: Records) -> list[dict]:
+        pass
+
+    @abstractmethod
+    async def async_save(self, records: Records) -> list[dict]:
+        pass
+
+    @abstractmethod
+    def _get(self, request: dict) -> list[dict]:
+        pass
+
+    @abstractmethod
+    async def _async_get(self, request: dict) -> list[dict]:
+        pass

splight_lib-5.10.1.dev0/splight_lib/client/datalake/constants.py

@@ -0,0 +1,31 @@
+from enum import Enum
+
+
+class StepName(str, Enum):
+    ADD_FIELDS = "addFields"
+    ADD_TO_SET = "addToSet"
+    BUCKET = "bucket"
+    BUCKET_AUTO = "bucketAuto"
+    BUCKET_GROUP = "bucketGroup"
+    COUNT = "count"
+    FACET = "facet"
+    GRAPH_LOOKUP = "graphLookup"
+    GROUP = "group"
+    INDEX_STATS = "indexStats"
+    LIMIT = "limit"
+    LIST_SESSIONS = "listSessions"
+    LOOKUP = "lookup"
+    MATCH = "match"
+    MERGE = "merge"
+    OUT = "out"
+    PLAN_CACHE_STATS = "planCacheStats"
+    PROJECT = "project"
+    REDACT = "redact"
+    REPLACE_ROOT = "replaceRoot"
+    REPLACE_WITH = "replaceWith"
+    SAMPLE = "sample"
+    SET = "set"
+    SKIP = "skip"
+    SORT = "sort"
+    UNSET = "unset"
+    UNWIND = "unwind"

splight_lib-5.10.1.dev0/splight_lib/client/datalake/exceptions.py

@@ -0,0 +1,12 @@
+class DatalakeRequestError(Exception):
+    def __init__(self, status_code: int, message: str):
+        self._msg = f"Request failed with status code {status_code}: {message}"
+        super().__init__(self._msg)
+
+
+class InvalidCollectionName(Exception):
+    def __init__(self, collection: str):
+        self._msg = f"Collection {collection} is not a valid collection"
+
+    def __str__(self) -> str:
+        return self._msg

splight_lib-5.10.1.dev0/splight_lib/client/datalake/remote_client.py

@@ -0,0 +1,252 @@
+from threading import Lock, Thread
+from time import sleep
+
+from furl import furl
+from retry import retry
+
+from splight_lib.auth import SplightAuthToken
+from splight_lib.client.datalake.abstract import (
+    AbstractDatalakeClient,
+    Records,
+)
+from splight_lib.client.datalake.buffer import DatalakeDocumentBuffer
+from splight_lib.client.datalake.exceptions import DatalakeRequestError
+from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
+from splight_lib.constants import DL_BUFFER_SIZE, DL_BUFFER_TIMEOUT
+from splight_lib.logging._internal import LogTags, get_splight_logger
+from splight_lib.restclient import SplightRestClient
+
+logger = get_splight_logger()
+
+EXCEPTIONS = (*SPLIGHT_REQUEST_EXCEPTIONS, DatalakeRequestError)
+
+
+class SyncRemoteDatalakeClient(AbstractDatalakeClient):
+    _PREFIX = "/data"
+
+    def __init__(
+        self, base_url: str, access_id: str, secret_key: str, *args, **kwargs
+    ):
+        super().__init__()
+        self._base_url = furl(base_url)
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+        self._restclient = SplightRestClient()
+        self._restclient.update_headers(token.header)
+        logger.debug(
+            "Remote datalake client initialized.", tags=LogTags.DATALAKE
+        )
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def save(self, records: Records) -> list[dict]:
+        url = self._base_url / f"{self._PREFIX}/write"
+        response = self._restclient.post(url, json=records)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def async_save(
+        self,
+        records: Records,
+    ) -> list[dict]:
+        # POST /data/write
+        url = self._base_url / f"{self._PREFIX}/write"
+        response = await self._restclient.async_post(url, json=records)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _get(self, request: dict) -> list[dict]:
+        url = self._base_url / f"{self._PREFIX}/read"
+        response = self._restclient.post(url, json=request)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return response.json()
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def _async_get(self, request: dict) -> list[dict]:
+        url = self._base_url / f"{self._PREFIX}/read"
+        response = await self._restclient.async_post(url, json=request)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return response.json()
+
+
+class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
+    _PREFIX = "data"
+
+    def __init__(
+        self,
+        base_url: str,
+        access_id: str,
+        secret_key: str,
+        buffer_size: int = DL_BUFFER_SIZE,
+        buffer_timeout: float = DL_BUFFER_TIMEOUT,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            base_url,
+            access_id,
+            secret_key,
+            buffer_size,
+            buffer_timeout,
+            *args,
+            **kwargs,
+        )
+        self._base_url = furl(base_url)
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+        self._restclient = SplightRestClient()
+        self._restclient.update_headers(token.header)
+
+        logger.debug(
+            "Initializing buffer with size %s and timeout %s",
+            buffer_size,
+            buffer_timeout,
+        )
+        self._data_buffers = {
+            "default": DatalakeDocumentBuffer(buffer_size, buffer_timeout),
+            "routine_evaluations": DatalakeDocumentBuffer(
+                buffer_size, buffer_timeout
+            ),
+        }
+        self._lock = Lock()
+        self._flush_thread = Thread(target=self._flusher, daemon=True)
+        self._flush_thread.start()
+        logger.debug(
+            "Buffered Remote datalake client initialized.",
+            tags=LogTags.DATALAKE,
+        )
+
+    def save(self, records: Records) -> list[dict]:
+        logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
+        collection = records["collection"]
+        instances = records["records"]
+        buffer = self._data_buffers[collection]
+        with self._lock:
+            if buffer.should_flush():
+                logger.debug(
+                    "Flushing datalake buffer with %s elements",
+                    len(buffer.data),
+                )
+                self._send_documents(collection, buffer.data)
+                buffer.reset()
+            buffer.add_documents(instances)
+        return instances
+
+    def _flusher(self):
+        while True:
+            for collection, buffer in self._data_buffers.items():
+                self._flush_buffer(collection, buffer)
+            sleep(0.5)
+
+    def _flush_buffer(
+        self, collection: str, buffer: DatalakeDocumentBuffer
+    ) -> None:
+        with self._lock:
+            if buffer.should_flush():
+                try:
+                    logger.debug(
+                        "Flushing datalake buffer with %s elements",
+                        len(buffer.data),
+                    )
+                    self._send_documents(collection, buffer.data)
+                    buffer.reset()
+                except Exception:
+                    logger.error("Unable to save documents", exc_info=True)
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _send_documents(self, collection: str, docs: list[dict]) -> list[dict]:
+        url = self._base_url / f"{self._PREFIX}/write"
+        data = {
+            "collection": collection,
+            "records": docs,
+        }
+        response = self._restclient.post(url, json=data)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return docs
+
+
+class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
+    _PREFIX = "data"
+
+    def __init__(
+        self,
+        base_url: str,
+        access_id: str,
+        secret_key: str,
+        buffer_size: int = DL_BUFFER_SIZE,
+        buffer_timeout: float = DL_BUFFER_TIMEOUT,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            base_url,
+            access_id,
+            secret_key,
+            buffer_size,
+            buffer_timeout,
+            *args,
+            **kwargs,
+        )
+        self._base_url = furl(base_url)
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+        self._restclient = SplightRestClient()
+        self._restclient.update_headers(token.header)
+
+        logger.debug(
+            "Initializing buffer with size %s and timeout %s",
+            buffer_size,
+            buffer_timeout,
+            tags=LogTags.DATALAKE,
+        )
+        self._data_buffers = {
+            "default": DatalakeDocumentBuffer(buffer_size, buffer_timeout),
+            "routine_evaluations": DatalakeDocumentBuffer(
+                buffer_size, buffer_timeout
+            ),
+        }
+        self._lock = Lock()
+        logger.debug(
+            "Synchronous Buffered Remote datalake client initialized.",
+            tags=LogTags.DATALAKE,
+        )
+
+    def save(self, records: Records) -> list[dict]:
+        logger.debug("Saving documents in datalake", tag=LogTags.DATALAKE)
+        collection = records["collection"]
+        buffer = self._data_buffers[collection]
+        with self._lock:
+            buffer.add_documents(records["records"])
+            if buffer.should_flush():
+                logger.debug(
+                    "Flushing datalake buffer with %s elements",
+                    len(buffer.data),
+                    tags=LogTags.DATALAKE,
+                )
+                self._send_documents(collection, buffer.data)
+                buffer.reset()
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _send_documents(self, collection: str, docs: list[dict]) -> list[dict]:
+        url = self._base_url / f"{self._PREFIX}/write"
+        data = {
+            "collection": collection,
+            "records": docs,
+        }
+        response = self._restclient.post(url, json=data)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return docs

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/client/exceptions.py

@@ -40,4 +40,5 @@ class InstanceNotFound(Exception):
         return self._msg
 
 
-class InvalidModel(Exception): ...
+class InvalidModel(Exception):
+    pass

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/client/tests/test_datalake.py

@@ -65,6 +65,7 @@ def test_save(mocker: MockerFixture):
     )
     collection = "collection_name"
     instances = [{"key": "value"}]
-    result = client.save(collection=collection, instances=instances)
+    records = {"collection": collection, "records": instances}
+    result = client.save(records)
     mock_post.assert_called_once()
     assert result == instances

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/component/spec.py

@@ -9,7 +9,6 @@ from splight_lib.component.exceptions import (
     ParameterDependencyError,
 )
 from splight_lib.constants import DESCRIPTION_MAX_LENGTH
-from splight_lib.models.base import SplightDatalakeBaseModel
 from splight_lib.models.component import (
     Component,
     ComponentObjectInstance,
@@ -22,6 +21,7 @@ from splight_lib.models.component import (
     Routine,
     get_field_value,
 )
+from splight_lib.models.datalake_base import SplightDatalakeBaseModel
 from splight_lib.utils.custom_model import create_custom_model
 
 VALID_PARAMETER_VALUES = {

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/execution/task.py

@@ -10,7 +10,8 @@ from splight_lib.execution.trigger import IntervalTrigger
 
 class BaseTask(ABC):
     @abstractmethod
-    def as_job(self) -> Dict: ...
+    def as_job(self) -> Dict:
+        pass
 
 
 class PeriodicTask(BaseTask):

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/__init__.py

@@ -18,6 +18,7 @@ from splight_lib.models.dashboard import (
     Filter,
     Tab,
 )
+from splight_lib.models.datalake import DataRequest, PipelineStep, Trace
 from splight_lib.models.file import File
 from splight_lib.models.function import Function, FunctionItem, QueryFilter
 from splight_lib.models.hub import HubComponent
@@ -46,6 +47,7 @@ __all__ = [
     ComponentObject,
     ComponentObjectInstance,
     Dashboard,
+    DataRequest,
     File,
     Filter,
     Function,
@@ -56,6 +58,7 @@ __all__ = [
     QueryFilter,
     Metadata,
     Number,
+    PipelineStep,
     RoutineEvaluation,
     Secret,
     Server,
@@ -65,4 +68,5 @@ __all__ = [
     RoutineObject,
     RoutineObjectInstance,
     Tab,
+    Trace,
 ]

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/actions.py

@@ -1,6 +1,9 @@
 from datetime import datetime
 
-from splight_lib.models.base import ResourceSummary, SplightDatabaseBaseModel
+from splight_lib.models.database_base import (
+    ResourceSummary,
+    SplightDatabaseBaseModel,
+)
 
 
 class SetPoint(SplightDatabaseBaseModel):

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/alert.py

@@ -14,7 +14,7 @@ from strenum import LowercaseStrEnum, UppercaseStrEnum
 from typing_extensions import TypedDict
 
 from splight_lib.constants import DESCRIPTION_MAX_LENGTH
-from splight_lib.models.base import SplightDatabaseBaseModel
+from splight_lib.models.database_base import SplightDatabaseBaseModel
 from splight_lib.models.exceptions import (
     InvalidAlertConfiguration,
     MissingAlertItemExpression,

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/asset.py

@@ -6,7 +6,10 @@ from pydantic import BaseModel, Field
 
 from splight_lib.constants import DESCRIPTION_MAX_LENGTH
 from splight_lib.models.attribute import Attribute
-from splight_lib.models.base import ResourceSummary, SplightDatabaseBaseModel
+from splight_lib.models.database_base import (
+    ResourceSummary,
+    SplightDatabaseBaseModel,
+)
 from splight_lib.models.exceptions import MethodNotAllowed
 from splight_lib.models.metadata import Metadata
 from splight_lib.models.tag import Tag

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/attribute.py

@@ -3,7 +3,7 @@ from typing import Optional
 
 from strenum import PascalCaseStrEnum
 
-from splight_lib.models.base import SplightDatabaseBaseModel
+from splight_lib.models.database_base import SplightDatabaseBaseModel
 
 
 class AttributeType(PascalCaseStrEnum):

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/component.py

@@ -20,11 +20,9 @@ from splight_lib.constants import DESCRIPTION_MAX_LENGTH
 from splight_lib.execution.scheduling import Crontab
 from splight_lib.models.asset import Asset
 from splight_lib.models.attribute import Attribute
-from splight_lib.models.base import (
-    SplightDatabaseBaseModel,
-    SplightDatalakeBaseModel,
-)
 from splight_lib.models.data_address import DataAddresses as DLDataAddress
+from splight_lib.models.database_base import SplightDatabaseBaseModel
+from splight_lib.models.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models.exceptions import InvalidObjectInstance
 from splight_lib.models.file import File
 from splight_lib.models.secret import Secret

{splight_lib-5.9.6.dev2 → splight_lib-5.10.1.dev0}/splight_lib/models/dashboard.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
 from pydantic import Field
 
 from splight_lib.constants import DESCRIPTION_MAX_LENGTH
-from splight_lib.models.base import SplightDatabaseBaseModel
+from splight_lib.models.database_base import SplightDatabaseBaseModel
 
 
 class Filter(SplightDatabaseBaseModel):
@@ -39,9 +39,7 @@ class ChartItem(SplightDatabaseBaseModel):
     aggregate_period: Optional[str] = None
     source: Optional[str] = None
     source_label: Optional[str] = None
-    source_type: Optional[str] = (
-        None  # TODO: define options (component, native)
-    )
+    source_type: Optional[str] = None
     source_component_label: Optional[str] = None
     source_component_id: Optional[str] = None
     output_format: Optional[str] = None

splight_lib-5.10.1.dev0/splight_lib/models/database_base.py

@@ -0,0 +1,79 @@
+from enum import auto
+from pathlib import Path
+from typing import Dict, List, Optional, TypeVar
+
+from pydantic import BaseModel, PrivateAttr
+from strenum import LowercaseStrEnum
+
+from splight_lib.client.database import DatabaseClientBuilder
+from splight_lib.client.database.abstract import AbstractDatabaseClient
+from splight_lib.settings import settings
+
+FilePath = TypeVar("FilePath", str, Path)
+
+
+class PrivacyPolicy(LowercaseStrEnum):
+    PUBLIC = auto()
+    PRIVATE = auto()
+
+
+class ResourceSummary(BaseModel):
+    id: str | None = None
+    name: str
+
+
+class SplightDatabaseBaseModel(BaseModel):
+    _db_client: AbstractDatabaseClient = PrivateAttr()
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        self._db_client = self.__get_database_client()
+
+    @staticmethod
+    def get_event_name(type_: str, action: str) -> str:
+        return f"{type_.lower()}-{action.lower()}"
+
+    def save(self):
+        files_dict = self._get_model_files_dict()
+        saved = self._db_client.save(
+            self.__class__.__name__,
+            self.model_dump(exclude_none=True, mode="json"),
+            files=files_dict,
+        )
+        instance = self.model_validate(saved)
+        for key, field in self.model_fields.items():
+            setattr(self, key, getattr(instance, key))
+
+    def delete(self):
+        self._db_client.delete(
+            resource_name=self.__class__.__name__, id=self.id
+        )
+
+    @classmethod
+    def retrieve(cls, resource_id: str) -> "SplightDatabaseBaseModel":
+        db_client = cls.__get_database_client()
+        instance = db_client.get(
+            resource_name=cls.__name__, id=resource_id, first=True
+        )
+        return cls.model_validate(instance) if instance else None
+
+    @classmethod
+    def list(cls, **params: Dict) -> List["SplightDatabaseBaseModel"]:
+        db_client = cls.__get_database_client()
+        instances = db_client.get(resource_name=cls.__name__, **params)
+        instances = [cls.model_validate(item) for item in instances]
+        return instances
+
+    @staticmethod
+    def __get_database_client() -> AbstractDatabaseClient:
+        db_client = DatabaseClientBuilder.build(
+            parameters={
+                "base_url": settings.SPLIGHT_PLATFORM_API_HOST,
+                "access_id": settings.SPLIGHT_ACCESS_ID,
+                "secret_key": settings.SPLIGHT_SECRET_KEY,
+            },
+        )
+        return db_client
+
+    def _get_model_files_dict(self) -> Optional[Dict]:
+        return None