hydroserverpy 1.2.1__tar.gz → 1.3.0__tar.gz

{hydroserverpy-1.2.1/src/hydroserverpy.egg-info → hydroserverpy-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.2.1
+Version: 1.3.0
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.2.1 → hydroserverpy-1.3.0}/README.md

@@ -33,7 +33,7 @@ from hydroserverpy import HydroServer
 # Initialize HydroServer connection with credentials.
 hs_api = HydroServer(
     host='https://playground.hydroserver.org',
-    username='user@example.com',
+    email='user@example.com',
     password='******'
 )
 ```

{hydroserverpy-1.2.1 → hydroserverpy-1.3.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = 1.2.1
+version = 1.3.0
 
 [options]
 package_dir = 

{hydroserverpy-1.2.1 → hydroserverpy-1.3.0}/src/hydroserverpy/__init__.py

@@ -1,4 +1,4 @@
-from .api.main import HydroServer
+from .api.client import HydroServer
 from .etl.hydroserver_etl import HydroServerETL
 from .quality import HydroServerQualityControl
 

hydroserverpy-1.2.1/src/hydroserverpy/api/main.py → hydroserverpy-1.3.0/src/hydroserverpy/api/client.py

@@ -1,8 +1,9 @@
 import requests
+import json
 from typing import Optional, Tuple
-from hydroserverpy.api.http import raise_for_hs_status
 from hydroserverpy.api.services import (
     WorkspaceService,
+    RoleService,
     ThingService,
     ObservedPropertyService,
     UnitService,
@@ -20,11 +21,14 @@ class HydroServer:
     def __init__(
         self,
         host: str,
+        auth_route: str = "/api/auth",
+        base_route: str = "/api/data",
         email: Optional[str] = None,
         password: Optional[str] = None,
         apikey: Optional[str] = None,
     ):
         self.host = host.strip("/")
+        self.base_route = base_route
         self.auth = (
             (
                 email or "__key__",
@@ -34,7 +38,7 @@ class HydroServer:
             else None
         )
 
-        self._auth_url = f"{self.host}/api/auth/app/session"
+        self._auth_url = f"{self.host}{auth_route}/app/session"
 
         self._session = None
         self._timeout = 60
@@ -52,6 +56,30 @@ class HydroServer:
 
         self._session.delete(self._auth_url, timeout=self._timeout)
 
+    def request(self, method, path, *args, **kwargs) -> requests.Response:
+        """Sends a request to HydroServer's API."""
+
+        for attempt in range(2):
+            try:
+                response = getattr(self._session, method)(
+                    f"{self.host}/{path.strip('/')}",
+                    timeout=self._timeout,
+                    *args,
+                    **kwargs,
+                )
+                self._raise_for_hs_status(response)
+            except (
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectionError,
+            ) as e:
+                if attempt == 0:
+                    self._init_session()
+                    continue
+                else:
+                    raise e
+
+            return response
+
     def _init_session(self, auth: Optional[Tuple[str, str]] = None) -> None:
         if self._session is not None:
             self.logout()
@@ -82,29 +110,25 @@ class HydroServer:
 
         return session_token
 
-    def request(self, method, path, *args, **kwargs) -> requests.Response:
-        """Sends a request to HydroServer's API."""
-
-        for attempt in range(2):
+    @staticmethod
+    def _raise_for_hs_status(response):
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
             try:
-                response = getattr(self._session, method)(
-                    f"{self.host}/{path.strip('/')}",
-                    timeout=self._timeout,
-                    *args,
-                    **kwargs,
+                http_error_msg = (
+                    f"{response.status_code} Client Error: "
+                    f"{str(json.loads(response.content).get('detail'))}"
                 )
-                raise_for_hs_status(response)
             except (
-                requests.exceptions.HTTPError,
-                requests.exceptions.ConnectionError,
-            ) as e:
-                if attempt == 0:
-                    self._init_session()
-                    continue
-                else:
-                    raise e
-
-            return response
+                    ValueError,
+                    TypeError,
+            ):
+                http_error_msg = e
+            if 400 <= response.status_code < 500:
+                raise requests.HTTPError(http_error_msg, response=response)
+            else:
+                raise requests.HTTPError(str(e), response=response)
 
     @property
     def workspaces(self):
@@ -112,6 +136,12 @@ class HydroServer:
 
         return WorkspaceService(self)
 
+    @property
+    def roles(self):
+        """Utilities for managing HydroServer workspaces."""
+
+        return RoleService(self)
+
     @property
     def things(self):
         """Utilities for managing HydroServer things."""

{hydroserverpy-1.2.1 → hydroserverpy-1.3.0}/src/hydroserverpy/api/models/__init__.py

@@ -5,6 +5,7 @@ from .iam.collaborator import Collaborator
 from .iam.apikey import APIKey
 from .iam.account import Account
 from .sta.datastream import Datastream
+from .sta.observation import ObservationCollection
 from .sta.observed_property import ObservedProperty
 from .sta.processing_level import ProcessingLevel
 from .sta.result_qualifier import ResultQualifier
@@ -19,5 +20,3 @@ Workspace.model_rebuild()
 Role.model_rebuild()
 Collaborator.model_rebuild()
 APIKey.model_rebuild()
-
-Unit.model_rebuild()

hydroserverpy-1.3.0/src/hydroserverpy/api/models/base.py

@@ -0,0 +1,207 @@
+import uuid
+from typing import Type, List, Dict, Any, Optional, ClassVar, TYPE_CHECKING
+from requests import Response
+from dataclasses import dataclass, field
+from pydantic import BaseModel, ConfigDict, PrivateAttr, Field
+from pydantic.alias_generators import to_camel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.services.base import HydroServerBaseService
+
+
+class HydroServerBaseModel(BaseModel):
+    uid: Optional[uuid.UUID] = Field(..., alias="id")
+
+    _client: "HydroServer" = PrivateAttr()
+    _service: "HydroServerBaseService" = PrivateAttr()
+    _server_data: Dict[str, Any] = PrivateAttr()
+    _editable_fields: ClassVar[set[str]] = set()
+
+    def __init__(self, *, client: "HydroServer", service: Optional["HydroServerBaseService"] = None, **data):
+        super().__init__(**data)
+
+        self._client = client
+        self._service = service
+        self._server_data = self.dict(by_alias=False).copy()
+
+    @classmethod
+    def get_route(cls):
+        raise NotImplementedError("Route not defined")
+
+    @property
+    def client(self) -> "HydroServer":
+        return self._client
+
+    @property
+    def service(self) -> "HydroServerBaseService":
+        return self._service
+
+    @property
+    def unsaved_changes(self) -> dict:
+        return {
+            k: v for k, v in self.__dict__.items()
+            if k in self._editable_fields and k in self._server_data and v != self._server_data[k]
+        }
+
+    def save(self):
+        """Saves changes to this resource to HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Saving not enabled for this object.")
+
+        if not self.uid:
+            raise AttributeError("Data cannot be saved: UID is not set.")
+
+        if self.unsaved_changes:
+            saved_resource = self.service.update(
+                self.uid, **self.unsaved_changes
+            )
+            self._server_data = saved_resource.dict(by_alias=False).copy()
+            self.__dict__.update(saved_resource.__dict__)
+
+    def refresh(self):
+        """Refreshes this resource from HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Refreshing not enabled for this object.")
+
+        if self.uid is None:
+            raise ValueError("Cannot refresh data without a valid ID.")
+
+        refreshed_resource = self.service.get(self.uid)
+        self._server_data = refreshed_resource.dict(by_alias=False).copy()
+        self.__dict__.update(refreshed_resource.__dict__)
+
+    def delete(self):
+        """Deletes this resource from HydroServer."""
+
+        if not self.service:
+            raise NotImplementedError("Deleting not enabled for this object.")
+
+        if self.uid is None:
+            raise AttributeError("Cannot delete data without a valid ID.")
+
+        self.service.delete(self.uid)
+        self.uid = None
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        populate_by_name=True,
+        str_strip_whitespace=True,
+        alias_generator=to_camel,
+    )
+
+
+@dataclass
+class HydroServerCollection:
+    items: List["HydroServerBaseModel"]
+    filters: Optional[dict[str, Any]] = None
+    order_by: Optional[List[str]] = None
+    page: Optional[int] = None
+    page_size: Optional[int] = None
+    total_pages: Optional[int] = None
+    total_count: Optional[int] = None
+
+    _service: Optional["HydroServerBaseService"] = field(init=False, repr=False)
+
+    def __init__(
+        self,
+        model: Type["HydroServerBaseModel"],
+        client: "HydroServer",
+        service: Optional["HydroServerBaseService"] = None,
+        response: Optional[Response] = None,
+        **data
+    ):
+        self._service = service
+
+        self.filters = data.get("filters")
+        self.order_by = data.get("order_by")
+        self.page = data.get("page") or (int(response.headers.get("X-Page")) if response else None)
+        self.page_size = data.get("page_size") or (int(response.headers.get("X-Page-Size")) if response else None)
+        self.total_pages = data.get("total_pages") or (int(response.headers.get("X-Total-Pages")) if response else None)
+        self.total_count = data.get("total_count") or (int(response.headers.get("X-Total-Count")) if response else None)
+
+        if "items" in data:
+            self.items = data["items"]
+        elif response is not None:
+            self.items = [model(client=client, **entity) for entity in response.json()]
+        else:
+            self.items = []
+
+    @property
+    def service(self) -> "HydroServerBaseService":
+        return self._service
+
+    def next_page(self):
+        """Fetches the next page of data from HydroServer."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        return self._service.list(
+            **(self.filters or {}),
+            page=(self.page or 0) + 1,
+            page_size=self.page_size or 100,
+            order_by=self.order_by or ...
+        )
+
+    def previous_page(self):
+        """Fetches the previous page of data from HydroServer."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        if not self.page or self.page <= 1:
+            return None
+
+        return self._service.list(
+            **(self.filters or {}),
+            page=self.page - 1,
+            page_size=self.page_size or 100,
+            order_by=self.order_by or ...
+        )
+
+    def fetch_all(self) -> "HydroServerCollection":
+        """Fetches all pages of data from HydroServer for this collection."""
+
+        if not self._service:
+            raise NotImplementedError("Pagination not enabled for this collection.")
+
+        all_items = []
+        current_page = self.page or 1
+        page_size = self.page_size or 100
+        total_pages = self.total_pages
+
+        page_num = 1
+        while total_pages is None or page_num <= total_pages:
+            if page_num == current_page:
+                all_items.extend(self.items)
+            else:
+                page = self._service.list(
+                    **(self.filters or {}),
+                    page=page_num,
+                    page_size=page_size,
+                    order_by=self.order_by or ...
+                )
+                if not page.items:
+                    break
+                all_items.extend(page.items)
+
+                if page.total_pages is not None:
+                    total_pages = page.total_pages
+
+            page_num += 1
+
+        return self.__class__(
+            model=type(self.items[0]) if self.items else None,
+            client=self.items[0].client if self.items else None,
+            service=self._service,
+            items=all_items,
+            filters=self.filters,
+            order_by=self.order_by,
+            page=1,
+            page_size=len(all_items),
+            total_pages=1,
+            total_count=len(all_items)
+        )

hydroserverpy-1.3.0/src/hydroserverpy/api/models/etl/data_archive.py

@@ -0,0 +1,77 @@
+import uuid
+from typing import Union, ClassVar, Optional, TYPE_CHECKING, List
+from pydantic import Field
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataArchive(
+    HydroServerBaseModel, OrchestrationConfigurationFields
+):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+    orchestration_system_id: uuid.UUID
+    workspace_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name", "settings", "interval", "interval_units", "crontab", "start_time", "end_time", "last_run_successful",
+        "last_run_message", "last_run", "next_run", "paused"
+    }
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.dataarchives, **data)
+
+        self._workspace = None
+        self._orchestration_system = None
+        self._datastreams = None
+
+    @classmethod
+    def get_route(cls):
+        return "data-archives"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data archive belongs to."""
+
+        if self._workspace is None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data archive."""
+
+        if self._orchestration_system is None:
+            self._orchestration_system = self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+
+        return self._orchestration_system
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data archive provides data for."""
+
+        if self._datastreams is None:
+            self._datastreams = self.client.datastreams.list(data_archive=self.uid, fetch_all=True).items
+
+        return self._datastreams
+
+    def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Add a datastream to this data archive."""
+
+        self.client.dataarchives.add_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Remove a datastream from this data archive."""
+
+        self.client.dataarchives.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )

hydroserverpy-1.3.0/src/hydroserverpy/api/models/etl/data_source.py

@@ -0,0 +1,111 @@
+import uuid
+import tempfile
+import requests
+from typing import Union, ClassVar, Optional, TYPE_CHECKING, List
+from pydantic import Field
+from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
+from .orchestration_system import OrchestrationSystem
+from .orchestration_configuration import OrchestrationConfigurationFields
+from ..sta.datastream import Datastream
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataSource(
+    HydroServerBaseModel, OrchestrationConfigurationFields
+):
+    name: str = Field(..., max_length=255)
+    settings: Optional[dict] = None
+    orchestration_system_id: uuid.UUID
+    workspace_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name", "settings", "interval", "interval_units", "crontab", "start_time", "end_time", "last_run_successful",
+        "last_run_message", "last_run", "next_run", "paused"
+    }
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.datasources, **data)
+
+        self._workspace = None
+        self._orchestration_system = None
+        self._datastreams = None
+
+    @classmethod
+    def get_route(cls):
+        return "data-sources"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data source belongs to."""
+
+        if self._workspace is None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def orchestration_system(self) -> "OrchestrationSystem":
+        """The orchestration system that manages this data source."""
+
+        if self._orchestration_system is None:
+            self._orchestration_system = self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+
+        return self._orchestration_system
+
+    @property
+    def datastreams(self) -> List["Datastream"]:
+        """The datastreams this data source provides data for."""
+
+        if self._datastreams is None:
+            self._datastreams = self.client.datastreams.list(data_source=self.uid, fetch_all=True).items
+
+        return self._datastreams
+
+    def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Add a datastream to this data source."""
+
+        self.client.datasources.add_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Remove a datastream from this data source."""
+
+        self.client.datasources.remove_datastream(
+            uid=self.uid, datastream=datastream
+        )
+
+    # TODO: Replace with ETL module.
+    def load_data(self):
+        """Load data for this data source."""
+
+        if self.paused is True:
+            return
+
+        if self.settings["extractor"]["type"] == "local":
+            with open(self.settings["extractor"]["sourceUri"]) as data_file:
+                loader = HydroServerETLCSV(
+                    self.client, data_file=data_file, data_source=self
+                )
+                loader.run()
+        elif self.settings["extractor"]["type"] == "HTTP":
+            with tempfile.NamedTemporaryFile(mode="w+") as temp_file:
+                response = requests.get(
+                    self.settings["extractor"]["sourceUri"],
+                    stream=True,
+                    timeout=60,
+                )
+                response.raise_for_status()
+                chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        temp_file.write(chunk.decode("utf-8"))
+                temp_file.seek(0)
+                loader = HydroServerETLCSV(
+                    self.client, data_file=temp_file, data_source=self
+                )
+                loader.run()

hydroserverpy-1.3.0/src/hydroserverpy/api/models/etl/orchestration_system.py

@@ -0,0 +1,63 @@
+import uuid
+from typing import Optional, ClassVar, List, TYPE_CHECKING
+from pydantic import BaseModel, Field
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, DataSource, DataArchive
+
+
+class OrchestrationSystemFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+
+
+class OrchestrationSystem(HydroServerBaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+    workspace_id: Optional[uuid.UUID] = None
+
+    _editable_fields: ClassVar[set[str]] = {"name", "orchestration_system_type"}
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.orchestrationsystems, **data)
+
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    @classmethod
+    def get_route(cls):
+        return "orchestration-systems"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this orchestration system belongs to."""
+
+        if self._workspace is None and self.workspace_id:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def datasources(self) -> List["DataSource"]:
+        """The data sources associated with this workspace."""
+
+        if self._datasources is None:
+            self._datasources = self.client.datasources.list(
+                orchestration_system=self.uid, fetch_all=True
+            ).items
+
+        return self._datasources
+
+    @property
+    def dataarchives(self) -> List["DataArchive"]:
+        """The data archives associated with this workspace."""
+
+        if self._dataarchives is None:
+            self._dataarchives = self.client.dataarchives.list(
+                orchestration_system=self.uid, fetch_all=True
+            ).items
+
+        return self._dataarchives