hydroserverpy 1.5.1__py3-none-any.whl

hydroserverpy/__init__.py

@@ -0,0 +1,7 @@
+from .api.client import HydroServer
+from .quality import HydroServerQualityControl
+
+__all__ = [
+    "HydroServer",
+    "HydroServerQualityControl",
+]

hydroserverpy/api/client.py

@@ -0,0 +1,203 @@
+import requests
+import json
+from typing import Optional, Tuple
+from hydroserverpy.api.services import (
+    WorkspaceService,
+    RoleService,
+    ThingService,
+    ObservedPropertyService,
+    UnitService,
+    ProcessingLevelService,
+    ResultQualifierService,
+    SensorService,
+    DatastreamService,
+    OrchestrationSystemService,
+    DataSourceService,
+    DataArchiveService,
+)
+
+
+class HydroServer:
+    def __init__(
+        self,
+        host: str,
+        auth_route: str = "/api/auth",
+        base_route: str = "/api/data",
+        email: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+    ):
+        self.host = host.strip("/")
+        self.base_route = base_route
+        self.auth = (
+            (
+                email or "__key__",
+                password or apikey,
+            )
+            if (email and password) or apikey
+            else None
+        )
+
+        self._auth_url = f"{self.host}{auth_route}/app/session"
+
+        self._session = None
+        self._timeout = 60
+        self._auth_header = None
+
+        self._init_session()
+
+    def login(self, email: str, password: str) -> None:
+        """Provide your HydroServer credentials to log in to your account."""
+
+        self._init_session(auth=(email, password))
+
+    def logout(self) -> None:
+        """End your HydroServer session."""
+
+        self._session.delete(self._auth_url, timeout=self._timeout)
+
+    def request(self, method, path, *args, **kwargs) -> requests.Response:
+        """Sends a request to HydroServer's API."""
+
+        for attempt in range(2):
+            try:
+                response = getattr(self._session, method)(
+                    f"{self.host}/{path.strip('/')}",
+                    timeout=self._timeout,
+                    *args,
+                    **kwargs,
+                )
+                self._raise_for_hs_status(response)
+            except (
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectionError,
+            ) as e:
+                if attempt == 0:
+                    self._init_session()
+                    continue
+                else:
+                    raise e
+
+            return response
+
+    def _init_session(self, auth: Optional[Tuple[str, str]] = None) -> None:
+        if self._session is not None:
+            self.logout()
+            self._session.close()
+
+        self._session = requests.Session()
+
+        auth = auth or self.auth
+
+        if auth and auth[0] == "__key__":
+            self._session.headers.update({"X-API-Key": auth[1]})
+        elif auth:
+            self._session.headers.update(
+                {"Authorization": f"Bearer {self._authenticate(auth[0], auth[1])}"}
+            )
+
+    def _authenticate(self, email: str, password: str) -> None:
+        response = self._session.post(
+            self._auth_url,
+            json={"email": email, "password": password},
+            timeout=self._timeout,
+        )
+        response.raise_for_status()
+        session_token = response.json().get("meta", {}).get("session_token")
+
+        if not session_token:
+            raise ValueError("Authentication failed: No access token returned.")
+
+        return session_token
+
+    @staticmethod
+    def _raise_for_hs_status(response):
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            try:
+                http_error_msg = (
+                    f"{response.status_code} Client Error: "
+                    f"{str(json.loads(response.content).get('detail'))}"
+                )
+            except (
+                    ValueError,
+                    TypeError,
+            ):
+                http_error_msg = e
+            if 400 <= response.status_code < 500:
+                raise requests.HTTPError(http_error_msg, response=response)
+            else:
+                raise requests.HTTPError(str(e), response=response)
+
+    @property
+    def workspaces(self):
+        """Utilities for managing HydroServer workspaces."""
+
+        return WorkspaceService(self)
+
+    @property
+    def roles(self):
+        """Utilities for managing HydroServer workspaces."""
+
+        return RoleService(self)
+
+    @property
+    def things(self):
+        """Utilities for managing HydroServer things."""
+
+        return ThingService(self)
+
+    @property
+    def observedproperties(self):
+        """Utilities for managing HydroServer observed properties."""
+
+        return ObservedPropertyService(self)
+
+    @property
+    def units(self):
+        """Utilities for managing HydroServer units."""
+
+        return UnitService(self)
+
+    @property
+    def processinglevels(self):
+        """Utilities for managing HydroServer processing levels."""
+
+        return ProcessingLevelService(self)
+
+    @property
+    def resultqualifiers(self):
+        """Utilities for managing HydroServer result qualifiers."""
+
+        return ResultQualifierService(self)
+
+    @property
+    def sensors(self):
+        """Utilities for managing HydroServer sensors."""
+
+        return SensorService(self)
+
+    @property
+    def datastreams(self):
+        """Utilities for managing HydroServer datastreams."""
+
+        return DatastreamService(self)
+
+    @property
+    def orchestrationsystems(self):
+        """Utilities for managing HydroServer orchestration systems."""
+
+        return OrchestrationSystemService(self)
+
+    @property
+    def datasources(self):
+        """Utilities for managing HydroServer data sources."""
+
+        return DataSourceService(self)
+
+    @property
+    def dataarchives(self):
+        """Utilities for managing HydroServer data archives."""
+
+        return DataArchiveService(self)

hydroserverpy/api/models/__init__.py

@@ -0,0 +1,22 @@
+from .iam.account import Account
+from .iam.workspace import Workspace
+from .iam.role import Role
+from .iam.collaborator import Collaborator
+from .iam.apikey import APIKey
+from .iam.account import Account
+from .sta.datastream import Datastream
+from .sta.observation import ObservationCollection
+from .sta.observed_property import ObservedProperty
+from .sta.processing_level import ProcessingLevel
+from .sta.result_qualifier import ResultQualifier
+from .sta.sensor import Sensor
+from .sta.thing import Thing
+from .sta.unit import Unit
+from .etl.orchestration_system import OrchestrationSystem
+from .etl.data_source import DataSource
+from .etl.data_archive import DataArchive
+
+Workspace.model_rebuild()
+Role.model_rebuild()
+Collaborator.model_rebuild()
+APIKey.model_rebuild()

hydroserverpy/api/models/base.py

@@ -0,0 +1,207 @@
+import uuid
+from typing import Type, List, Dict, Any, Optional, ClassVar, TYPE_CHECKING
+from requests import Response
+from dataclasses import dataclass, field
+from pydantic import BaseModel, ConfigDict, PrivateAttr, Field
+from pydantic.alias_generators import to_camel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.services.base import HydroServerBaseService
+
+
+class HydroServerBaseModel(BaseModel):
+    uid: Optional[uuid.UUID] = Field(..., alias="id")
+
+    _client: "HydroServer" = PrivateAttr()
+    _service: "HydroServerBaseService" = PrivateAttr()
+    _server_data: Dict[str, Any] = PrivateAttr()
+    _editable_fields: ClassVar[set[str]] = set()
+
+    def __init__(self, *, client: "HydroServer", service: Optional["HydroServerBaseService"] = None, **data):
+        super().__init__(**data)
+
+        self._client = client
+        self._service = service
+        self._server_data = self.dict(by_alias=False).copy()
+
+    @classmethod
+    def get_route(cls):
+        raise NotImplementedError("Route not defined")
+
+    @property
+    def client(self) -> "HydroServer":
+        return self._client
+
+    @property
+    def service(self) -> "HydroServerBaseService":
+        return self._service
+
+    @property
+    def unsaved_changes(self) -> dict:
+        return {
+            k: v for k, v in self.__dict__.items()
+            if k in self._editable_fields and k in self._server_data and v != self._server_data[k]
+        }
+
+    def save(self):
+        """Saves changes to this resource to HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Saving not enabled for this object.")
+
+        if not self.uid:
+            raise AttributeError("Data cannot be saved: UID is not set.")
+
+        if self.unsaved_changes:
+            saved_resource = self.service.update(
+                self.uid, **self.unsaved_changes
+            )
+            self._server_data = saved_resource.dict(by_alias=False).copy()
+            self.__dict__.update(saved_resource.__dict__)
+
+    def refresh(self):
+        """Refreshes this resource from HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Refreshing not enabled for this object.")
+
+        if self.uid is None:
+            raise ValueError("Cannot refresh data without a valid ID.")
+
+        refreshed_resource = self.service.get(self.uid)
+        self._server_data = refreshed_resource.dict(by_alias=False).copy()
+        self.__dict__.update(refreshed_resource.__dict__)
+
+    def delete(self):
+        """Deletes this resource from HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Deleting not enabled for this object.")
+
+        if self.uid is None:
+            raise AttributeError("Cannot delete data without a valid ID.")
+
+        self.service.delete(self.uid)
+        self.uid = None
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        populate_by_name=True,
+        str_strip_whitespace=True,
+        alias_generator=to_camel,
+    )
+
+
+@dataclass
+class HydroServerCollection:
+    items: List["HydroServerBaseModel"]
+    filters: Optional[dict[str, Any]] = None
+    order_by: Optional[List[str]] = None
+    page: Optional[int] = None
+    page_size: Optional[int] = None
+    total_pages: Optional[int] = None
+    total_count: Optional[int] = None
+
+    _service: Optional["HydroServerBaseService"] = field(init=False, repr=False)
+
+    def __init__(
+        self,
+        model: Type["HydroServerBaseModel"],
+        client: "HydroServer",
+        service: Optional["HydroServerBaseService"] = None,
+        response: Optional[Response] = None,
+        **data
+    ):
+        self._service = service
+
+        self.filters = data.get("filters")
+        self.order_by = data.get("order_by")
+        self.page = data.get("page") or (int(response.headers.get("X-Page")) if response else None)
+        self.page_size = data.get("page_size") or (int(response.headers.get("X-Page-Size")) if response else None)
+        self.total_pages = data.get("total_pages") or (int(response.headers.get("X-Total-Pages")) if response else None)
+        self.total_count = data.get("total_count") or (int(response.headers.get("X-Total-Count")) if response else None)
+
+        if "items" in data:
+            self.items = data["items"]
+        elif response is not None:
+            self.items = [model(client=client, **entity) for entity in response.json()]
+        else:
+            self.items = []
+
+    @property
+    def service(self) -> "HydroServerBaseService":
+        return self._service
+
+    def next_page(self):
+        """Fetches the next page of data from HydroServer."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        return self._service.list(
+            **(self.filters or {}),
+            page=(self.page or 0) + 1,
+            page_size=self.page_size or 100,
+            order_by=self.order_by or ...
+        )
+
+    def previous_page(self):
+        """Fetches the previous page of data from HydroServer."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        if not self.page or self.page <= 1:
+            return None
+
+        return self._service.list(
+            **(self.filters or {}),
+            page=self.page - 1,
+            page_size=self.page_size or 100,
+            order_by=self.order_by or ...
+        )
+
+    def fetch_all(self) -> "HydroServerCollection":
+        """Fetches all pages of data from HydroServer for this collection."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        all_items = []
+        current_page = self.page or 1
+        page_size = self.page_size or 100
+        total_pages = self.total_pages
+
+        page_num = 1
+        while total_pages is None or page_num <= total_pages:
+            if page_num == current_page:
+                all_items.extend(self.items)
+            else:
+                page = self._service.list(
+                    **(self.filters or {}),
+                    page=page_num,
+                    page_size=page_size,
+                    order_by=self.order_by or ...
+                )
+                if not page.items:
+                    break
+                all_items.extend(page.items)
+
+                if page.total_pages is not None:
+                    total_pages = page.total_pages
+
+            page_num += 1
+
+        return self.__class__(
+            model=type(self.items[0]) if self.items else None,
+            client=self.items[0].client if self.items else None,
+            service=self._service,
+            items=all_items,
+            filters=self.filters,
+            order_by=self.order_by,
+            page=1,
+            page_size=len(all_items),
+            total_pages=1,
+            total_count=len(all_items)
+        )

hydroserverpy/api/models/etl/__init__.py

@@ -0,0 +1,26 @@
+from .extractors import Extractor, HTTPExtractor, LocalFileExtractor, FTPExtractor
+from .transformers import JSONTransformer, CSVTransformer, Transformer
+from .loaders import HydroServerLoader, Loader
+
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .orchestration_system import OrchestrationSystem
+from .data_source import DataSource
+
+__all__ = [
+    "CSVTransformer",
+    "JSONTransformer",
+    "LocalFileExtractor",
+    "FTPExtractor",
+    "HTTPExtractor",
+    "Extractor",
+    "Transformer",
+    "Loader",
+    "HydroServerLoader",
+    "EtlConfiguration",
+    "Schedule",
+    "Status",
+    "OrchestrationSystem",
+    "DataSource",
+]

hydroserverpy/api/models/etl/data_archive.py

@@ -0,0 +1,77 @@
+import uuid
+from typing import Union, ClassVar, Optional, TYPE_CHECKING, List
+from pydantic import Field
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataArchive(
+    HydroServerBaseModel, OrchestrationConfigurationFields
+):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+    orchestration_system_id: uuid.UUID
+    workspace_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name", "settings", "interval", "interval_units", "crontab", "start_time", "end_time", "last_run_successful",
+        "last_run_message", "last_run", "next_run", "paused"
+    }
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.dataarchives, **data)
+
+        self._workspace = None
+        self._orchestration_system = None
+        self._datastreams = None
+
+    @classmethod
+    def get_route(cls):
+        return "data-archives"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data archive belongs to."""
+
+        if self._workspace is None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data archive."""
+
+        if self._orchestration_system is None:
+            self._orchestration_system = self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+
+        return self._orchestration_system
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data archive provides data for."""
+
+        if self._datastreams is None:
+            self._datastreams = self.client.datastreams.list(data_archive=self.uid, fetch_all=True).items
+
+        return self._datastreams
+
+    def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Add a datastream to this data archive."""
+
+        self.client.dataarchives.add_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Remove a datastream from this data archive."""
+
+        self.client.dataarchives.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )

hydroserverpy/api/models/etl/data_source.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+from datetime import datetime, timedelta, timezone
+from functools import cached_property
+import logging
+import uuid
+from typing import ClassVar, TYPE_CHECKING, List, Optional, Union
+import croniter
+import pandas as pd
+from pydantic import Field
+
+from ..base import HydroServerBaseModel
+from ..sta.datastream import Datastream
+from .orchestration_system import OrchestrationSystem
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .factories import extractor_factory, transformer_factory, loader_factory
+from .loaders import HydroServerLoader
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataSource(HydroServerBaseModel):
+    name: str = Field(..., max_length=255)
+    settings: EtlConfiguration
+    orchestration_system_id: uuid.UUID
+    schedule: Schedule
+    status: Status
+    workspace_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name",
+        "settings",
+        "status",
+        "schedule",
+        "interval",
+        "interval_units",
+        "crontab",
+        "start_time",
+        "end_time",
+        "last_run_successful",
+        "last_run_message",
+        "last_run",
+        "next_run",
+        "paused",
+    }
+
+    def __init__(self, client: HydroServer, **data):
+        super().__init__(client=client, service=client.datasources, **data)
+
+    @classmethod
+    def get_route(cls):
+        return "data-sources"
+
+    @cached_property
+    def workspace(self) -> Workspace:
+        return self.client.workspaces.get(uid=self.workspace_id)
+
+    @cached_property
+    def orchestration_system(self) -> OrchestrationSystem:
+        return self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+
+    @cached_property
+    def datastreams(self) -> List[Datastream]:
+        return self.client.datastreams.list(data_source=self.uid, fetch_all=True).items
+
+    # TODO: Add functions like add_payload, add_mapping, etc. and don't allow the user to manually
+    # link or unlink datastreams - handle that automatically.
+    def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Add a datastream to this data source."""
+
+        self.client.datasources.add_datastream(uid=self.uid, datastream=datastream)
+
+    def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Remove a datastream from this data source."""
+
+        self.client.datasources.remove_datastream(uid=self.uid, datastream=datastream)
+
+    def _next_run(self) -> Optional[str]:
+        now = datetime.now(timezone.utc)
+        if cron := self.schedule.crontab:
+            return croniter.croniter(cron, now).get_next(datetime).isoformat()
+        if iv := self.schedule.interval:
+            unit = self.schedule.interval_units or "minutes"
+            return (now + timedelta(**{unit: iv})).isoformat()
+        return None
+
+    def _update_status(self, loader: HydroServerLoader, success: bool, msg: str):
+        short_msg = msg if len(msg) <= 255 else msg[:252] + "…"
+        loader.client.datasources.update(
+            uid=self.uid,
+            last_run=datetime.now(timezone.utc).isoformat(),
+            last_run_successful=success,
+            last_run_message=short_msg,
+            next_run=self._next_run(),
+        )
+
+    def is_empty(self, data):
+        if data is None:
+            return True
+        if isinstance(data, pd.DataFrame) and data.empty:
+            return True
+        return False
+
+    def load_data(self, payload_name: str = None):
+        """Load data for this data source."""
+        if self.status.paused is True:
+            return
+
+        if payload_name:
+            self.load_data_for_payload(payload_name)
+        else:
+            for p in self.settings.payloads:
+                self.load_data_for_payload(p.name)
+
+    def load_data_for_payload(self, payload_name: str):
+        payload = next(p for p in self.settings.payloads if p.name == payload_name)
+
+        extractor_cls = extractor_factory(self.settings.extractor)
+        transformer_cls = transformer_factory(self.settings.transformer)
+        loader_cls = loader_factory(self.settings.loader, self.client, self.uid)
+
+        try:
+            logging.info("Starting extract")
+            data = extractor_cls.extract(payload, loader_cls)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the extractor"
+                )
+                return
+
+            logging.info("Starting transform")
+            data = transformer_cls.transform(data, payload.mappings)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the transformer"
+                )
+                return
+
+            logging.info("Starting load")
+            loader_cls.load(data, payload)
+            self._update_status(loader_cls, True, "OK")
+        except Exception as e:
+            self._update_status(loader_cls, False, str(e))