hydroserverpy 0.5.0b1__tar.gz → 0.5.0b2__tar.gz

{hydroserverpy-0.5.0b1/src/hydroserverpy.egg-info → hydroserverpy-0.5.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 0.5.0b1
+Version: 0.5.0b2
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = 0.5.0b1
+version = 0.5.0b2
 
 [options]
 package_dir = 

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/http.py

@@ -3,8 +3,6 @@ from requests import HTTPError
 
 
 def raise_for_hs_status(response):
-    """"""
-
     try:
         response.raise_for_status()
     except HTTPError as e:

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/main.py

@@ -10,6 +10,9 @@ from hydroserverpy.api.services import (
     ResultQualifierService,
     SensorService,
     DatastreamService,
+    OrchestrationSystemService,
+    DataSourceService,
+    DataArchiveService,
 )
 
 
@@ -150,3 +153,21 @@ class HydroServer:
         """Utilities for managing HydroServer datastreams."""
 
         return DatastreamService(self)
+
+    @property
+    def orchestrationsystems(self):
+        """Utilities for managing HydroServer orchestration systems."""
+
+        return OrchestrationSystemService(self)
+
+    @property
+    def datasources(self):
+        """Utilities for managing HydroServer data sources."""
+
+        return DataSourceService(self)
+
+    @property
+    def dataarchives(self):
+        """Utilities for managing HydroServer data archives."""
+
+        return DataArchiveService(self)

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/models/__init__.py

@@ -10,6 +10,9 @@ from .sta.result_qualifier import ResultQualifier
 from .sta.sensor import Sensor
 from .sta.thing import Thing
 from .sta.unit import Unit
+from .etl.orchestration_system import OrchestrationSystem
+from .etl.data_source import DataSource
+from .etl.data_archive import DataArchive
 
 Workspace.model_rebuild()
 Role.model_rebuild()

hydroserverpy-0.5.0b2/src/hydroserverpy/api/models/etl/data_archive.py

@@ -0,0 +1,105 @@
+from typing import Union, Optional, TYPE_CHECKING, List
+from uuid import UUID
+from pydantic import BaseModel, Field
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataArchiveFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+
+
+class DataArchive(
+    HydroServerModel, DataArchiveFields, OrchestrationConfigurationFields
+):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="dataarchives", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._orchestration_system_id = str(
+            data.get("orchestration_system_id") or data["orchestrationSystem"]["id"]
+        )
+
+        self._workspace = None
+
+        if data.get("orchestrationSystem"):
+            self._orchestration_system = OrchestrationSystem(
+                _connection=_connection,
+                _uid=self._orchestration_system_id,
+                **data["orchestrationSystem"]
+            )
+        else:
+            self._orchestration_system = None
+
+        if data.get("datastreams"):
+            self._datastreams = [
+                Datastream(_connection=_connection, _uid=datastream["id"], **datastream)
+                for datastream in data["datastreams"]
+            ]
+        else:
+            self._datastreams = []
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data archive belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data archive."""
+
+        if self._orchestration_system is None and self._orchestration_system_id:
+            self._orchestration_system = self._connection.orchestration_systems.get(
+                uid=self._orchestration_system_id
+            )
+
+        return self._orchestration_system
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data archive provides data for."""
+
+        return self._datastreams
+
+    def refresh(self):
+        """Refresh this data archive from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+
+    def save(self):
+        """Save changes to this data archive to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this data archive from HydroServer."""
+
+        super()._delete()
+
+    def add_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Add a datastream to this data archive."""
+
+        self._connection.dataarchives.add_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    def remove_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Remove a datastream from this data archive."""
+
+        self._connection.dataarchives.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )

hydroserverpy-0.5.0b2/src/hydroserverpy/api/models/etl/data_source.py

@@ -0,0 +1,150 @@
+import tempfile
+from typing import Union, List, Optional, TYPE_CHECKING
+from uuid import UUID
+from pydantic import BaseModel, Field
+from urllib.request import urlopen
+from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataSourceFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+
+
+class DataSource(HydroServerModel, DataSourceFields, OrchestrationConfigurationFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="datasources", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._orchestration_system_id = str(
+            data.get("orchestration_system_id") or data["orchestrationSystem"]["id"]
+        )
+
+        self._workspace = None
+
+        if data.get("orchestrationSystem"):
+            self._orchestration_system = OrchestrationSystem(
+                _connection=_connection,
+                _uid=self._orchestration_system_id,
+                **data["orchestrationSystem"]
+            )
+        else:
+            self._orchestration_system = None
+
+        if data.get("datastreams"):
+            self._datastreams = [
+                Datastream(_connection=_connection, _uid=datastream["id"], **datastream)
+                for datastream in data["datastreams"]
+            ]
+        else:
+            self._datastreams = []
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data source belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data source."""
+
+        if self._orchestration_system is None and self._orchestration_system_id:
+            self._orchestration_system = self._connection.orchestration_systems.get(
+                uid=self._orchestration_system_id
+            )
+
+        return self._orchestration_system
+
+    @orchestration_system.setter
+    def orchestration_system(
+        self, orchestration_system: Union["OrchestrationSystem", UUID, str]
+    ):
+        if not orchestration_system:
+            raise ValueError("Orchestration system of data source cannot be None.")
+        if str(getattr(orchestration_system, "uid", orchestration_system)) != str(
+            self.orchestration_system.uid
+        ):
+            self._orchestration_system = self._connection.orchestrationsystems.get(
+                uid=str(getattr(orchestration_system, "uid", orchestration_system))
+            )
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data source provides data for."""
+
+        if self._datastreams is None:
+            data_source = self._connection.datasources.get(uid=self.uid)
+            self._datastreams = data_source.datastreams
+
+        return self._datastreams
+
+    def refresh(self):
+        """Refresh this data source from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+        self._datastreams = None
+
+    def save(self):
+        """Save changes to this data source to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this data source from HydroServer."""
+
+        super()._delete()
+
+    def add_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Add a datastream to this data source."""
+
+        self._connection.datasources.add_datastream(uid=self.uid, datastream=datastream)
+
+    def remove_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Remove a datastream from this data source."""
+
+        self._connection.datasources.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    # TODO: Replace with ETL module.
+    def load_data(self):
+        """Load data for this data source."""
+
+        if self.paused is True:
+            return
+
+        if self.settings["extractor"]["type"] == "local":
+            with open(self.settings["extractor"]["path"]) as data_file:
+                loader = HydroServerETLCSV(
+                    self._connection, data_file=data_file, data_source=self
+                )
+                loader.run()
+        elif self.settings["extractor"]["type"] == "HTTP":
+            with tempfile.NamedTemporaryFile(mode="w") as temp_file:
+                with urlopen(self.settings["extractor"]["urlTemplate"]) as response:
+                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        temp_file.write(chunk)
+                temp_file.seek(0)
+                loader = HydroServerETLCSV(
+                    self._connection, data_file=temp_file, data_source=self
+                )
+                loader.run()

hydroserverpy-0.5.0b2/src/hydroserverpy/api/models/etl/orchestration_configuration.py

@@ -0,0 +1,35 @@
+from pydantic import AliasPath
+from typing import Optional, Literal
+from datetime import datetime
+from pydantic import BaseModel, Field
+
+
+class OrchestrationConfigurationFields(BaseModel):
+    interval: Optional[int] = Field(
+        None, gt=0, validation_alias=AliasPath("schedule", "interval")
+    )
+    interval_units: Optional[Literal["minutes", "hours", "days"]] = Field(
+        None, validation_alias=AliasPath("schedule", "intervalUnits")
+    )
+    crontab: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("schedule", "crontab")
+    )
+    start_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "startTime")
+    )
+    end_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "endTime")
+    )
+    last_run_successful: Optional[bool] = Field(
+        None, validation_alias=AliasPath("status", "lastRunSuccessful")
+    )
+    last_run_message: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("status", "lastRunMessage")
+    )
+    last_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "lastRun")
+    )
+    next_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "nextRun")
+    )
+    paused: bool = Field(False, validation_alias=AliasPath("status", "paused"))

hydroserverpy-0.5.0b2/src/hydroserverpy/api/models/etl/orchestration_system.py

@@ -0,0 +1,78 @@
+from typing import Union, List, TYPE_CHECKING
+from uuid import UUID
+from pydantic import BaseModel, Field
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, DataSource, DataArchive
+
+
+class OrchestrationSystemFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+
+
+class OrchestrationSystem(HydroServerModel, OrchestrationSystemFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection,
+            _model_ref="orchestrationsystems",
+            _uid=_uid,
+            **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this orchestration system belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def datasources(self) -> List["DataSource"]:
+        """The data sources associated with this workspace."""
+
+        if self._datasources is None:
+            self._datasources = self._connection.datasources.list(
+                orchestration_system=self.uid
+            )
+
+        return self._datasources
+
+    @property
+    def dataarchives(self) -> List["DataArchive"]:
+        """The data archives associated with this workspace."""
+
+        if self._dataarchives is None:
+            self._dataarchives = self._connection.dataarchives.list(
+                orchestration_system=self.uid
+            )
+
+        return self._dataarchives
+
+    def refresh(self):
+        """Refresh this orchestration system from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    def save(self):
+        """Save changes to this orchestration system to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this orchestration system from HydroServer."""
+
+        super()._delete()

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/models/iam/workspace.py

@@ -16,6 +16,9 @@ if TYPE_CHECKING:
         ProcessingLevel,
         ResultQualifier,
         Datastream,
+        OrchestrationSystem,
+        DataSource,
+        DataArchive,
     )
 
 
@@ -41,6 +44,9 @@ class Workspace(HydroServerModel, WorkspaceFields):
         self._units = None
         self._sensors = None
         self._datastreams = None
+        self._orchestrationsystems = None
+        self._datasources = None
+        self._dataarchives = None
 
     @property
     def roles(self) -> List["Role"]:
@@ -131,6 +137,35 @@ class Workspace(HydroServerModel, WorkspaceFields):
 
         return self._datastreams
 
+    @property
+    def orchestrationsystems(self) -> List["OrchestrationSystem"]:
+        """The orchestration systems associated with this workspace."""
+
+        if self._orchestrationsystems is None:
+            self._orchestrationsystems = self._connection.orchestrationsystems.list(
+                workspace=self.uid
+            )
+
+        return self._orchestrationsystems
+
+    @property
+    def datasources(self) -> List["DataSource"]:
+        """The data sources associated with this workspace."""
+
+        if self._datasources is None:
+            self._datasources = self._connection.datasources.list(workspace=self.uid)
+
+        return self._datasources
+
+    @property
+    def dataarchives(self) -> List["DataArchive"]:
+        """The data archives associated with this workspace."""
+
+        if self._dataarchives is None:
+            self._dataarchives = self._connection.dataarchives.list(workspace=self.uid)
+
+        return self._dataarchives
+
     def refresh(self) -> None:
         """Refresh the workspace details from HydroServer."""
 

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/models/sta/datastream.py

@@ -263,7 +263,9 @@ class Datastream(HydroServerModel, DatastreamFields):
         """The processing level of this datastream."""
 
         if self._processing_level is None:
-            self._processing_level = self._connection.processinglevels.get(uid=self._processing_level_id)
+            self._processing_level = self._connection.processinglevels.get(
+                uid=self._processing_level_id
+            )
             self._original_data["processing_level"] = self._processing_level
 
         return self._processing_level

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/services/__init__.py

@@ -6,3 +6,6 @@ from .sta.processing_level import ProcessingLevelService
 from .sta.result_qualifier import ResultQualifierService
 from .sta.sensor import SensorService
 from .sta.datastream import DatastreamService
+from .etl.orchestration_system import OrchestrationSystemService
+from .etl.data_source import DataSourceService
+from .etl.data_archive import DataArchiveService

{hydroserverpy-0.5.0b1 → hydroserverpy-0.5.0b2}/src/hydroserverpy/api/services/base.py

@@ -1,4 +1,5 @@
 from typing import TYPE_CHECKING, Type, Union, Optional
+from datetime import datetime
 from uuid import UUID
 
 if TYPE_CHECKING:
@@ -38,7 +39,7 @@ class EndpointService:
         path = f"/{self._api_route}/{self._endpoint_route}"
         headers = {"Content-type": "application/json"}
         response = self._connection.request(
-            "post", path, headers=headers, json=kwargs
+            "post", path, headers=headers, json=self._to_iso_time(kwargs)
         ).json()
 
         return self._model(
@@ -49,7 +50,7 @@ class EndpointService:
         path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}"
         headers = {"Content-type": "application/json"}
         response = self._connection.request(
-            "patch", path, headers=headers, json=kwargs
+            "patch", path, headers=headers, json=self._to_iso_time(kwargs)
         ).json()
 
         return self._model(
@@ -62,6 +63,15 @@ class EndpointService:
 
         return response
 
+    def _to_iso_time(self, obj):
+        if isinstance(obj, dict):
+            return {k: self._to_iso_time(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self._to_iso_time(i) for i in obj]
+        elif isinstance(obj, datetime):
+            return obj.isoformat()
+        return obj
+
 
 class SensorThingsService(EndpointService):
     _sta_route: str