hydroserverpy 0.4.0__py3-none-any.whl → 0.5.0b2__py3-none-any.whl

hydroserverpy/__init__.py

@@ -1,7 +1,6 @@
-from .core.service import HydroServer
-from .quality.service import HydroServerQualityControl
+from .api.main import HydroServer
 from .etl.hydroserver_etl import HydroServerETL
-
+from .quality import HydroServerQualityControl
 
 __all__ = [
     "HydroServer",

hydroserverpy/api/http.py

@@ -0,0 +1,22 @@
+import json
+from requests import HTTPError
+
+
+def raise_for_hs_status(response):
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        try:
+            http_error_msg = (
+                f"{response.status_code} Client Error: "
+                f"{str(json.loads(response.content).get('detail'))}"
+            )
+        except (
+            ValueError,
+            TypeError,
+        ):
+            http_error_msg = e
+        if 400 <= response.status_code < 500:
+            raise HTTPError(http_error_msg, response=response)
+        else:
+            raise HTTPError(str(e), response=response)

hydroserverpy/api/main.py

@@ -0,0 +1,173 @@
+import requests
+from typing import Optional, Tuple
+from hydroserverpy.api.http import raise_for_hs_status
+from hydroserverpy.api.services import (
+    WorkspaceService,
+    ThingService,
+    ObservedPropertyService,
+    UnitService,
+    ProcessingLevelService,
+    ResultQualifierService,
+    SensorService,
+    DatastreamService,
+    OrchestrationSystemService,
+    DataSourceService,
+    DataArchiveService,
+)
+
+
+class HydroServer:
+    def __init__(
+        self,
+        host: str,
+        email: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+    ):
+        self.host = host.strip("/")
+        self.auth = (
+            (
+                email or "__key__",
+                password or apikey,
+            )
+            if (email and password) or apikey
+            else None
+        )
+
+        self._auth_url = f"{self.host}/api/auth/app/session"
+
+        self._session = None
+        self._timeout = 60
+        self._auth_header = None
+
+        self._init_session()
+
+    def login(self, email: str, password: str) -> None:
+        """Provide your HydroServer credentials to log in to your account."""
+
+        self._init_session(auth=(email, password))
+
+    def logout(self) -> None:
+        """End your HydroServer session."""
+
+        self._session.delete(self._auth_url, timeout=self._timeout)
+
+    def _init_session(self, auth: Optional[Tuple[str, str]] = None) -> None:
+        if self._session is not None:
+            self.logout()
+            self._session.close()
+
+        self._session = requests.Session()
+
+        auth = auth or self.auth
+
+        if auth and auth[0] == "__key__":
+            self._session.headers.update({"key": auth[1]})
+        elif auth:
+            self._session.headers.update(
+                {"Authorization": f"Bearer {self._authenticate(auth[0], auth[1])}"}
+            )
+
+    def _authenticate(self, email: str, password: str) -> None:
+        response = self._session.post(
+            self._auth_url,
+            json={"email": email, "password": password},
+            timeout=self._timeout,
+        )
+        response.raise_for_status()
+        session_token = response.json().get("meta", {}).get("session_token")
+
+        if not session_token:
+            raise ValueError("Authentication failed: No access token returned.")
+
+        return session_token
+
+    def request(self, method, path, *args, **kwargs) -> requests.Response:
+        """Sends a request to HydroServer's API."""
+
+        for attempt in range(2):
+            try:
+                response = getattr(self._session, method)(
+                    f"{self.host}/{path.strip('/')}",
+                    timeout=self._timeout,
+                    *args,
+                    **kwargs,
+                )
+                raise_for_hs_status(response)
+            except (
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectionError,
+            ) as e:
+                if attempt == 0:
+                    self._init_session()
+                    continue
+                else:
+                    raise e
+
+            return response
+
+    @property
+    def workspaces(self):
+        """Utilities for managing HydroServer workspaces."""
+
+        return WorkspaceService(self)
+
+    @property
+    def things(self):
+        """Utilities for managing HydroServer things."""
+
+        return ThingService(self)
+
+    @property
+    def observedproperties(self):
+        """Utilities for managing HydroServer observed properties."""
+
+        return ObservedPropertyService(self)
+
+    @property
+    def units(self):
+        """Utilities for managing HydroServer units."""
+
+        return UnitService(self)
+
+    @property
+    def processinglevels(self):
+        """Utilities for managing HydroServer processing levels."""
+
+        return ProcessingLevelService(self)
+
+    @property
+    def resultqualifiers(self):
+        """Utilities for managing HydroServer result qualifiers."""
+
+        return ResultQualifierService(self)
+
+    @property
+    def sensors(self):
+        """Utilities for managing HydroServer sensors."""
+
+        return SensorService(self)
+
+    @property
+    def datastreams(self):
+        """Utilities for managing HydroServer datastreams."""
+
+        return DatastreamService(self)
+
+    @property
+    def orchestrationsystems(self):
+        """Utilities for managing HydroServer orchestration systems."""
+
+        return OrchestrationSystemService(self)
+
+    @property
+    def datasources(self):
+        """Utilities for managing HydroServer data sources."""
+
+        return DataSourceService(self)
+
+    @property
+    def dataarchives(self):
+        """Utilities for managing HydroServer data archives."""
+
+        return DataArchiveService(self)

hydroserverpy/api/models/__init__.py

@@ -0,0 +1,21 @@
+from .iam.account import Account
+from .iam.workspace import Workspace
+from .iam.role import Role
+from .iam.collaborator import Collaborator
+from .iam.account import Account
+from .sta.datastream import Datastream
+from .sta.observed_property import ObservedProperty
+from .sta.processing_level import ProcessingLevel
+from .sta.result_qualifier import ResultQualifier
+from .sta.sensor import Sensor
+from .sta.thing import Thing
+from .sta.unit import Unit
+from .etl.orchestration_system import OrchestrationSystem
+from .etl.data_source import DataSource
+from .etl.data_archive import DataArchive
+
+Workspace.model_rebuild()
+Role.model_rebuild()
+Collaborator.model_rebuild()
+
+Unit.model_rebuild()

hydroserverpy/api/models/base.py

@@ -0,0 +1,74 @@
+from typing import Optional
+from uuid import UUID
+from pydantic import BaseModel, PrivateAttr, ConfigDict, computed_field
+from pydantic.alias_generators import to_camel
+
+
+class HydroServerBaseModel(BaseModel):
+    _uid: Optional[UUID] = PrivateAttr()
+
+    def __init__(self, _uid: Optional[UUID] = None, **data):
+        super().__init__(**data)
+        self._uid = _uid
+
+    @computed_field
+    @property
+    def uid(self) -> Optional[UUID]:
+        """The unique identifier for this resource."""
+
+        return self._uid
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        populate_by_name=True,
+        str_strip_whitespace=True,
+        alias_generator=to_camel,
+    )
+
+
+class HydroServerModel(HydroServerBaseModel):
+    _model_ref: str = PrivateAttr()
+    _original_data: Optional[dict] = PrivateAttr()
+
+    def __init__(self, _connection, _model_ref, _uid: Optional[UUID] = None, **data):
+        if isinstance(_uid, str):
+            _uid = UUID(_uid)
+
+        super().__init__(_uid=_uid, **data)
+
+        self._connection = _connection
+        self._model_ref = _model_ref
+        self._original_data = self.dict(by_alias=False).copy()
+
+    @property
+    def _patch_data(self) -> dict:
+        return {
+            key: getattr(self, key)
+            for key, value in self._original_data.items()
+            if hasattr(self, key) and getattr(self, key) != value
+        }
+
+    def _refresh(self) -> None:
+        """Refresh this resource from HydroServer."""
+
+        self._original_data = (
+            getattr(self._connection, self._model_ref)
+            .get(uid=self.uid)
+            .model_dump(exclude=["uid"])
+        )
+        self.__dict__.update(self._original_data)
+
+    def _save(self) -> None:
+        if self._patch_data:
+            entity = getattr(self._connection, self._model_ref).update(
+                uid=self.uid, **self._patch_data
+            )
+            self._original_data = entity.dict(by_alias=False, exclude=["uid"])
+            self.__dict__.update(self._original_data)
+
+    def _delete(self) -> None:
+        if not self._uid:
+            raise AttributeError("This resource cannot be deleted: UID is not set.")
+
+        getattr(self._connection, self._model_ref).delete(uid=self._uid)
+        self._uid = None

hydroserverpy/api/models/etl/data_archive.py

@@ -0,0 +1,105 @@
+from typing import Union, Optional, TYPE_CHECKING, List
+from uuid import UUID
+from pydantic import BaseModel, Field
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataArchiveFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+
+
+class DataArchive(
+    HydroServerModel, DataArchiveFields, OrchestrationConfigurationFields
+):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="dataarchives", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._orchestration_system_id = str(
+            data.get("orchestration_system_id") or data["orchestrationSystem"]["id"]
+        )
+
+        self._workspace = None
+
+        if data.get("orchestrationSystem"):
+            self._orchestration_system = OrchestrationSystem(
+                _connection=_connection,
+                _uid=self._orchestration_system_id,
+                **data["orchestrationSystem"]
+            )
+        else:
+            self._orchestration_system = None
+
+        if data.get("datastreams"):
+            self._datastreams = [
+                Datastream(_connection=_connection, _uid=datastream["id"], **datastream)
+                for datastream in data["datastreams"]
+            ]
+        else:
+            self._datastreams = []
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data archive belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data archive."""
+
+        if self._orchestration_system is None and self._orchestration_system_id:
+            self._orchestration_system = self._connection.orchestration_systems.get(
+                uid=self._orchestration_system_id
+            )
+
+        return self._orchestration_system
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data archive provides data for."""
+
+        return self._datastreams
+
+    def refresh(self):
+        """Refresh this data archive from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+
+    def save(self):
+        """Save changes to this data archive to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this data archive from HydroServer."""
+
+        super()._delete()
+
+    def add_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Add a datastream to this data archive."""
+
+        self._connection.dataarchives.add_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    def remove_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Remove a datastream from this data archive."""
+
+        self._connection.dataarchives.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )

hydroserverpy/api/models/etl/data_source.py

@@ -0,0 +1,150 @@
+import tempfile
+from typing import Union, List, Optional, TYPE_CHECKING
+from uuid import UUID
+from pydantic import BaseModel, Field
+from urllib.request import urlopen
+from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataSourceFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+
+
+class DataSource(HydroServerModel, DataSourceFields, OrchestrationConfigurationFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="datasources", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._orchestration_system_id = str(
+            data.get("orchestration_system_id") or data["orchestrationSystem"]["id"]
+        )
+
+        self._workspace = None
+
+        if data.get("orchestrationSystem"):
+            self._orchestration_system = OrchestrationSystem(
+                _connection=_connection,
+                _uid=self._orchestration_system_id,
+                **data["orchestrationSystem"]
+            )
+        else:
+            self._orchestration_system = None
+
+        if data.get("datastreams"):
+            self._datastreams = [
+                Datastream(_connection=_connection, _uid=datastream["id"], **datastream)
+                for datastream in data["datastreams"]
+            ]
+        else:
+            self._datastreams = []
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data source belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data source."""
+
+        if self._orchestration_system is None and self._orchestration_system_id:
+            self._orchestration_system = self._connection.orchestration_systems.get(
+                uid=self._orchestration_system_id
+            )
+
+        return self._orchestration_system
+
+    @orchestration_system.setter
+    def orchestration_system(
+        self, orchestration_system: Union["OrchestrationSystem", UUID, str]
+    ):
+        if not orchestration_system:
+            raise ValueError("Orchestration system of data source cannot be None.")
+        if str(getattr(orchestration_system, "uid", orchestration_system)) != str(
+            self.orchestration_system.uid
+        ):
+            self._orchestration_system = self._connection.orchestrationsystems.get(
+                uid=str(getattr(orchestration_system, "uid", orchestration_system))
+            )
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data source provides data for."""
+
+        if self._datastreams is None:
+            data_source = self._connection.datasources.get(uid=self.uid)
+            self._datastreams = data_source.datastreams
+
+        return self._datastreams
+
+    def refresh(self):
+        """Refresh this data source from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+        self._datastreams = None
+
+    def save(self):
+        """Save changes to this data source to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this data source from HydroServer."""
+
+        super()._delete()
+
+    def add_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Add a datastream to this data source."""
+
+        self._connection.datasources.add_datastream(uid=self.uid, datastream=datastream)
+
+    def remove_datastream(self, datastream: Union["Datastream", UUID, str]):
+        """Remove a datastream from this data source."""
+
+        self._connection.datasources.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    # TODO: Replace with ETL module.
+    def load_data(self):
+        """Load data for this data source."""
+
+        if self.paused is True:
+            return
+
+        if self.settings["extractor"]["type"] == "local":
+            with open(self.settings["extractor"]["path"]) as data_file:
+                loader = HydroServerETLCSV(
+                    self._connection, data_file=data_file, data_source=self
+                )
+                loader.run()
+        elif self.settings["extractor"]["type"] == "HTTP":
+            with tempfile.NamedTemporaryFile(mode="w") as temp_file:
+                with urlopen(self.settings["extractor"]["urlTemplate"]) as response:
+                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        temp_file.write(chunk)
+                temp_file.seek(0)
+                loader = HydroServerETLCSV(
+                    self._connection, data_file=temp_file, data_source=self
+                )
+                loader.run()

hydroserverpy/api/models/etl/orchestration_configuration.py

@@ -0,0 +1,35 @@
+from pydantic import AliasPath
+from typing import Optional, Literal
+from datetime import datetime
+from pydantic import BaseModel, Field
+
+
+class OrchestrationConfigurationFields(BaseModel):
+    interval: Optional[int] = Field(
+        None, gt=0, validation_alias=AliasPath("schedule", "interval")
+    )
+    interval_units: Optional[Literal["minutes", "hours", "days"]] = Field(
+        None, validation_alias=AliasPath("schedule", "intervalUnits")
+    )
+    crontab: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("schedule", "crontab")
+    )
+    start_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "startTime")
+    )
+    end_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "endTime")
+    )
+    last_run_successful: Optional[bool] = Field(
+        None, validation_alias=AliasPath("status", "lastRunSuccessful")
+    )
+    last_run_message: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("status", "lastRunMessage")
+    )
+    last_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "lastRun")
+    )
+    next_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "nextRun")
+    )
+    paused: bool = Field(False, validation_alias=AliasPath("status", "paused"))

hydroserverpy/api/models/etl/orchestration_system.py

@@ -0,0 +1,78 @@
+from typing import Union, List, TYPE_CHECKING
+from uuid import UUID
+from pydantic import BaseModel, Field
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, DataSource, DataArchive
+
+
+class OrchestrationSystemFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+
+
+class OrchestrationSystem(HydroServerModel, OrchestrationSystemFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection,
+            _model_ref="orchestrationsystems",
+            _uid=_uid,
+            **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this orchestration system belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    @property
+    def datasources(self) -> List["DataSource"]:
+        """The data sources associated with this workspace."""
+
+        if self._datasources is None:
+            self._datasources = self._connection.datasources.list(
+                orchestration_system=self.uid
+            )
+
+        return self._datasources
+
+    @property
+    def dataarchives(self) -> List["DataArchive"]:
+        """The data archives associated with this workspace."""
+
+        if self._dataarchives is None:
+            self._dataarchives = self._connection.dataarchives.list(
+                orchestration_system=self.uid
+            )
+
+        return self._dataarchives
+
+    def refresh(self):
+        """Refresh this orchestration system from HydroServer."""
+
+        super()._refresh()
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    def save(self):
+        """Save changes to this orchestration system to HydroServer."""
+
+        super()._save()
+
+    def delete(self):
+        """Delete this orchestration system from HydroServer."""
+
+        super()._delete()

hydroserverpy/api/models/iam/account.py

@@ -0,0 +1,12 @@
+from typing import Optional
+from pydantic import BaseModel, Field, EmailStr
+
+
+class Account(BaseModel):
+    name: str = Field(..., max_length=255)
+    email: EmailStr
+    organization_name: Optional[str] = None
+    phone: Optional[str] = Field(None, max_length=15)
+    address: Optional[str] = Field(None, max_length=255)
+    link: Optional[str] = Field(None, max_length=2000)
+    user_type: str = Field(..., max_length=255, alias="type")