hydroserverpy 1.0.0__tar.gz → 1.1.0__tar.gz

{hydroserverpy-1.0.0/src/hydroserverpy.egg-info → hydroserverpy-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.0.0
+Version: 1.1.0
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = v1.0.0
+version = 1.1.0
 
 [options]
 package_dir = 

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/src/hydroserverpy/api/main.py

@@ -62,7 +62,7 @@ class HydroServer:
         auth = auth or self.auth
 
         if auth and auth[0] == "__key__":
-            self._session.headers.update({"key": auth[1]})
+            self._session.headers.update({"X-API-Key": auth[1]})
         elif auth:
             self._session.headers.update(
                 {"Authorization": f"Bearer {self._authenticate(auth[0], auth[1])}"}

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/src/hydroserverpy/api/models/__init__.py

@@ -2,6 +2,7 @@ from .iam.account import Account
 from .iam.workspace import Workspace
 from .iam.role import Role
 from .iam.collaborator import Collaborator
+from .iam.apikey import APIKey
 from .iam.account import Account
 from .sta.datastream import Datastream
 from .sta.observed_property import ObservedProperty
@@ -17,5 +18,6 @@ from .etl.data_archive import DataArchive
 Workspace.model_rebuild()
 Role.model_rebuild()
 Collaborator.model_rebuild()
+APIKey.model_rebuild()
 
 Unit.model_rebuild()

hydroserverpy-1.1.0/src/hydroserverpy/api/models/iam/apikey.py

@@ -0,0 +1,77 @@
+from typing import Optional, Union, TYPE_CHECKING
+from uuid import UUID
+from datetime import datetime
+from pydantic import BaseModel
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, Role
+
+
+class APIKeyFields(BaseModel):
+    name: str
+    description: Optional[str] = None
+    is_active: bool
+    expires_at: Optional[datetime] = None
+    role: "Role"
+
+
+class APIKey(HydroServerModel, APIKeyFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="apikeys", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._workspace = None
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data source belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    def refresh(self):
+        """Refresh this data source from HydroServer."""
+
+        self._original_data = (
+            self._connection.workspaces.get_api_key(
+                uid=self._workspace_id, api_key_id=self.uid
+            ).model_dump(exclude=["uid"])
+        )
+        self.__dict__.update(self._original_data)
+        self._workspace = None
+
+    def save(self):
+        """Save changes to this data source to HydroServer."""
+
+        if self._patch_data:
+            api_key = self._connection.workspaces.update_api_key(
+                uid=self._workspace_id, api_key_id=self.uid, **self._patch_data
+            )
+            self._original_data = api_key.dict(by_alias=False, exclude=["uid"])
+            self.__dict__.update(self._original_data)
+
+    def delete(self):
+        """Delete this data source from HydroServer."""
+
+        if not self._uid:
+            raise AttributeError("This resource cannot be deleted: UID is not set.")
+
+        self._connection.workspaces.delete_api_key(
+            uid=self._workspace_id, api_key_id=self.uid
+        )
+        self._uid = None
+
+    def regenerate(self):
+        """Regenerates this API key. WARNING: Previous key will be invalidated."""
+
+        _, key = self._connection.workspaces.regenerate_api_key(
+            uid=self._workspace_id, api_key_id=self.uid
+        )
+
+        return key

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/src/hydroserverpy/api/models/iam/workspace.py

@@ -1,5 +1,6 @@
 from typing import List, Union, Optional, TYPE_CHECKING
 from uuid import UUID
+from datetime import datetime
 from pydantic import BaseModel, Field, EmailStr
 from ..base import HydroServerModel
 
@@ -8,6 +9,7 @@ if TYPE_CHECKING:
     from hydroserverpy.api.models import (
         Role,
         Collaborator,
+        APIKey,
         Account,
         Thing,
         ObservedProperty,
@@ -189,6 +191,79 @@ class Workspace(HydroServerModel, WorkspaceFields):
 
         super()._delete()
 
+    def list_api_keys(self) -> List["APIKey"]:
+        """Get all API keys associated with this workspace."""
+
+        return self._connection.workspaces.list_api_keys(uid=self.uid)
+
+    def get_api_key(self, api_key: Union["APIKey", UUID, str]) -> "APIKey":
+        """Get an API key associated with this workspace."""
+
+        return self._connection.workspaces.get_api_key(
+            uid=self.uid,
+            api_key_id=str(getattr(api_key, "uid", api_key))
+        )
+
+    def create_api_key(
+        self,
+        role: Union["Role", UUID, str],
+        name: str,
+        description: Optional[str] = None,
+        is_active: bool = True,
+        expires_at: Optional[datetime] = None
+    ):
+        """Create an API key associated with this workspace."""
+
+        api_key, key = self._connection.workspaces.create_api_key(
+            uid=self.uid,
+            role=role,
+            name=name,
+            description=description,
+            is_active=is_active,
+            expires_at=expires_at
+        )
+
+        return api_key, key
+
+    def update_api_key(
+            self,
+            api_key_id: Union[UUID, str],
+            role: Union["Role", UUID, str] = ...,
+            name: str = ...,
+            description: Optional[str] = ...,
+            is_active: bool = ...,
+            expires_at: Optional[datetime] = ...
+    ):
+        """Create an API key associated with this workspace."""
+
+        return self._connection.workspaces.update_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id,
+            role=role,
+            name=name,
+            description=description,
+            is_active=is_active,
+            expires_at=expires_at
+        )
+
+    def delete_api_key(self, api_key_id: Union[UUID, str]):
+        """Delete an API key associated with this workspace."""
+
+        return self._connection.workspaces.delete_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id
+        )
+
+    def regenerate_api_key(self, api_key_id: Union[UUID, str]):
+        """Regenerate an API key associated with this workspace."""
+
+        api_key, key = self._connection.workspaces.regenerate_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id
+        )
+
+        return api_key, key
+
     def add_collaborator(
         self, email: EmailStr, role: Union["Role", UUID, str]
     ) -> "Collaborator":

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/src/hydroserverpy/api/services/iam/workspace.py

@@ -1,7 +1,8 @@
-from typing import TYPE_CHECKING, Union, List
+from typing import TYPE_CHECKING, Union, List, Tuple, Optional
 from pydantic import EmailStr
 from uuid import UUID
-from hydroserverpy.api.models import Workspace, Role, Collaborator
+from datetime import datetime
+from hydroserverpy.api.models import Workspace, Role, Collaborator, APIKey
 from ..base import EndpointService
 
 
@@ -107,6 +108,114 @@ class WorkspaceService(EndpointService):
         path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/collaborators"
         self._connection.request("delete", path, json={"email": email})
 
+    def list_api_keys(self, uid: Union[UUID, str]) -> List["APIKey"]:
+        """Get all API keys associated with a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys"
+        response = self._connection.request("get", path)
+
+        return [APIKey(_connection=self._connection, _uid=UUID(str(obj.pop("id"))), **obj) for obj in response.json()]
+
+    def get_api_key(self, uid: Union[UUID, str], api_key_id: Union[UUID, str]) -> "APIKey":
+        """Get an API key associated with a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{api_key_id}"
+        response = self._connection.request("get", path).json()
+
+        return APIKey(_connection=self._connection, _uid=UUID(str(response.pop("id"))), **response)
+
+    def create_api_key(
+        self,
+        uid: Union[UUID, str],
+        role: Union["Role", UUID, str],
+        name: str,
+        description: Optional[str] = None,
+        is_active: bool = True,
+        expires_at: Optional[datetime] = None
+    ) -> Tuple["APIKey", str]:
+        """Create an API key for a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys"
+        kwargs = {
+            "roleId": str(getattr(role, "uid", role)),
+            "name": name,
+            "description": description,
+            "isActive": is_active,
+            "expiresAt": expires_at
+        }
+        headers = {"Content-type": "application/json"}
+
+        response = self._connection.request(
+            "post", path, headers=headers, json=self._to_iso_time(kwargs)
+        ).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        ), response["key"]
+
+    def update_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str],
+        role: Union["Role", UUID, str] = ...,
+        name: str = ...,
+        description: Optional[str] = ...,
+        is_active: bool = ...,
+        expires_at: Optional[datetime] = ...
+    ) -> "APIKey":
+        """Update an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}"
+        kwargs = {
+            "roleId": ... if role is ... else str(getattr(role, "uid", role)),
+            "name": name,
+            "description": description,
+            "isActive": is_active,
+            "expiresAt": (
+                expires_at.isoformat()
+                if expires_at
+                not in (
+                    None,
+                    ...,
+                )
+                else None
+            )
+        }
+        headers = {"Content-type": "application/json"}
+
+        response = self._connection.request(
+            "patch", path, headers=headers,
+            json={k: v for k, v in kwargs.items() if v is not ...}
+        ).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        )
+
+    def delete_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str]
+    ):
+        """Delete an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}"
+        self._connection.request("delete", path)
+
+    def regenerate_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str]
+    ):
+        """Regenerate an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}/regenerate"
+        response = self._connection.request("put", path).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        ), response["key"]
+
     def transfer_ownership(self, uid: Union[UUID, str], email: str) -> None:
         """Transfer ownership of a workspace to another HydroServer user."""
 

hydroserverpy-1.1.0/src/hydroserverpy/etl/extractors/http_extractor.py

@@ -0,0 +1,99 @@
+import logging
+from hydroserverpy.etl.types import TimeRange
+import requests
+from io import BytesIO
+from typing import Dict
+from .base import Extractor
+
+
+class HTTPExtractor(Extractor):
+    def __init__(self, settings: object):
+        self.url = settings["urlTemplate"]
+        # self.url = self.format_url(url, url_variables or {})
+        # self.params = settings.get('params', )
+        # self.headers = headers
+        # self.auth = auth
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+        # TODO: Uncomment this once url templates work on in the Data Management App
+        # start_times = [
+        #     req["start_time"] for req in data_requirements.values() if req["start_time"]
+        # ]
+
+        # if start_times:
+        #     oldest_start_time = min(start_times)
+        #     start_time_key = self.params.pop("start_time_key", None)
+        #     if start_time_key:
+        #         self.params[start_time_key] = oldest_start_time
+        #         logging.info(
+        #             f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
+        #         )
+        #     else:
+        #         logging.warning("'start_time_key' not found in params.")
+
+        # end_times = [
+        #     req["end_time"] for req in data_requirements.values() if req["end_time"]
+        # ]
+
+        # if end_times:
+        #     newest_end_time = max(end_times)
+        #     end_time_key = self.params.pop("end_time_key", None)
+        #     if end_time_key:
+        #         self.params[end_time_key] = newest_end_time
+        #         logging.info(
+        #             f"Set end_time to {newest_end_time} and removed 'end_time_key'"
+        #         )
+        #     else:
+        #         logging.warning("'end_time_key' not found in params.")
+
+    def extract(self):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+
+        logging.info(f"Requesting data from → {self.url}")
+
+        # endpoints = [
+        #     "https://httpbin.org/get",
+        #     "https://jsonplaceholder.typicode.com/posts/1",
+        #     "https://api.github.com",
+        #     "https://api.ipify.org?format=json",
+        #     "https://www.python.org/",
+        #     "https://waterservices.usgs.gov/nwis/iv/?&format=json&sites=01646500&parameterCd=00060",
+        #     "https://datahub.io/core/country-list/r/data.csv",
+        #     "https://raw.githubusercontent.com/cs109/2014_data/master/countries.csv",
+        #     # "https://rain-flow.slco.org/export/file/?delimiter=comma&site_id=68&data_start=2025-04-09&data_end=2025-05-09&device_id=2",
+        #     # "https://rain-flow.slco.org/export/file/?mime=txt&delimiter=comma&site_id=68&data_start=2025-05-09%2000:00:00&data_end=2025-05-09%2023:59:59&device_id=2"
+        # ]
+        # for url in endpoints:
+        #     try:
+        #         r = requests.get(url, timeout=10)
+        #         print(f"{url:50} → {r.status_code}")
+        #     except Exception as e:
+        #         print(f"{url:50} → ERROR: {e}")
+
+        try:
+            response = requests.get(self.url)
+        except Exception as e:
+            logging.error(f"Failed to fetch {repr(self.url)}: {e}")
+            raise
+
+        logging.info(f"Received response")
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data
+
+    @staticmethod
+    def format_url(url_template, url_variables):
+        try:
+            url = url_template.format(**url_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing configuration url_variable: {missing_key}")
+
+        return url

hydroserverpy-1.1.0/src/hydroserverpy/etl/extractors/local_file_extractor.py

@@ -0,0 +1,29 @@
+import logging
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, settings: object):
+        if "path" not in settings:
+            message = "Missing required setting 'path' in LocalFileExtractor settings."
+            logging.error(message)
+            raise ValueError(message)
+        self.path = settings["path"]
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.path, "r")
+            logging.info(f"Successfully opened file '{self.path}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.path}': {e}")
+            return None

{hydroserverpy-1.0.0 → hydroserverpy-1.1.0}/src/hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -1,5 +1,8 @@
+import datetime
 from hydroserverpy import HydroServer
 from typing import Dict, Optional
+
+from src.hydroserverpy.etl.types import TimeRange
 from .base import Loader
 import logging
 import pandas as pd
@@ -13,20 +16,25 @@ class HydroServerLoader(HydroServer, Loader):
     def __init__(
         self,
         host: str,
-        username: Optional[str] = None,
+        email: Optional[str] = None,
         password: Optional[str] = None,
         apikey: Optional[str] = None,
-        api_route: str = "api",
     ):
-        super().__init__(host, username, password, apikey, api_route)
+        super().__init__(
+            host=host,
+            email=email,
+            password=password,
+            apikey=apikey,
+        )
 
-    def load(self, data: pd.DataFrame, source_target_map) -> None:
+    def load(self, data: pd.DataFrame, payload_settings) -> None:
         """
         Load observations from a DataFrame to the HydroServer.
 
         :param data: A Pandas DataFrame where each column corresponds to a datastream.
         """
-        data_requirements = self.get_data_requirements(source_target_map)
+        mappings = payload_settings["mappings"]
+        time_ranges = self.get_data_requirements(mappings)
         for ds_id in data.columns:
             if ds_id == "timestamp":
                 continue
@@ -35,9 +43,17 @@ class HydroServerLoader(HydroServer, Loader):
             df.rename(columns={ds_id: "value"}, inplace=True)
             df.dropna(subset=["value"], inplace=True)
 
-            phenomenon_end_time = data_requirements[ds_id]["start_time"]
-            if phenomenon_end_time:
-                df = df[df["timestamp"] > phenomenon_end_time]
+            # ensure the timestamp column is UTC‑aware
+            timestamp_column = df["timestamp"]
+            if timestamp_column.dt.tz is None:
+                df["timestamp"] = timestamp_column.dt.tz_localize("UTC")
+
+            time_range = time_ranges[ds_id]
+            start_ts = pd.to_datetime(time_range["start_time"], utc=True)
+
+            if start_ts:
+                df = df[df["timestamp"] > start_ts]
+            logging.info(f"start cutoff for data loading {start_ts}")
             if df.empty:
                 logging.warning(
                     f"No new data to upload for datastream {ds_id}. Skipping."
@@ -45,24 +61,31 @@ class HydroServerLoader(HydroServer, Loader):
                 continue
             self.datastreams.load_observations(uid=ds_id, observations=df)
 
-    def get_data_requirements(
-        self, source_target_map
-    ) -> Dict[str, Dict[str, pd.Timestamp]]:
+    def get_data_requirements(self, source_target_map) -> Dict[str, TimeRange]:
         """
         Each target system needs to be able to answer the question: 'What data do you need?'
         and return a time range for each target time series. Usually the answer will be
         'anything newer than my most recent observation'.
         """
         data_requirements = {}
-        for ds_id in source_target_map.values():
-            datastream = self.datastreams.get(uid=ds_id)
+        target_ids = [mapping["targetIdentifier"] for mapping in source_target_map]
+        for id in target_ids:
+            datastream = self.datastreams.get(uid=id)
             if not datastream:
                 message = "Couldn't fetch target datastream. ETL process aborted."
                 logging.error(message)
                 raise message
-            start_time = pd.Timestamp(
+
+            start_ts = pd.Timestamp(
                 datastream.phenomenon_end_time or "1970-01-01T00:00:00Z"
             )
-            end_time = pd.Timestamp.now()
-            data_requirements[ds_id] = {"start_time": start_time, "end_time": end_time}
+            if start_ts.tzinfo is None:
+                start_ts = start_ts.tz_localize("UTC")
+
+            end_ts = pd.Timestamp.now(tz="UTC")
+
+            data_requirements[id] = {
+                "start_time": start_ts.isoformat(),
+                "end_time": end_ts.isoformat(),
+            }
         return data_requirements