hydroserverpy 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

hydroserverpy/api/main.py

@@ -62,7 +62,7 @@ class HydroServer:
         auth = auth or self.auth
 
         if auth and auth[0] == "__key__":
-            self._session.headers.update({"key": auth[1]})
+            self._session.headers.update({"X-API-Key": auth[1]})
         elif auth:
             self._session.headers.update(
                 {"Authorization": f"Bearer {self._authenticate(auth[0], auth[1])}"}

hydroserverpy/api/models/__init__.py

@@ -2,6 +2,7 @@ from .iam.account import Account
 from .iam.workspace import Workspace
 from .iam.role import Role
 from .iam.collaborator import Collaborator
+from .iam.apikey import APIKey
 from .iam.account import Account
 from .sta.datastream import Datastream
 from .sta.observed_property import ObservedProperty
@@ -17,5 +18,6 @@ from .etl.data_archive import DataArchive
 Workspace.model_rebuild()
 Role.model_rebuild()
 Collaborator.model_rebuild()
+APIKey.model_rebuild()
 
 Unit.model_rebuild()

hydroserverpy/api/models/iam/apikey.py

@@ -0,0 +1,77 @@
+from typing import Optional, Union, TYPE_CHECKING
+from uuid import UUID
+from datetime import datetime
+from pydantic import BaseModel
+from ..base import HydroServerModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, Role
+
+
+class APIKeyFields(BaseModel):
+    name: str
+    description: Optional[str] = None
+    is_active: bool
+    expires_at: Optional[datetime] = None
+    role: "Role"
+
+
+class APIKey(HydroServerModel, APIKeyFields):
+    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
+        super().__init__(
+            _connection=_connection, _model_ref="apikeys", _uid=_uid, **data
+        )
+
+        self._workspace_id = str(data.get("workspace_id") or data["workspaceId"])
+        self._workspace = None
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this data source belongs to."""
+
+        if self._workspace is None and self._workspace_id:
+            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+
+        return self._workspace
+
+    def refresh(self):
+        """Refresh this data source from HydroServer."""
+
+        self._original_data = (
+            self._connection.workspaces.get_api_key(
+                uid=self._workspace_id, api_key_id=self.uid
+            ).model_dump(exclude=["uid"])
+        )
+        self.__dict__.update(self._original_data)
+        self._workspace = None
+
+    def save(self):
+        """Save changes to this data source to HydroServer."""
+
+        if self._patch_data:
+            api_key = self._connection.workspaces.update_api_key(
+                uid=self._workspace_id, api_key_id=self.uid, **self._patch_data
+            )
+            self._original_data = api_key.dict(by_alias=False, exclude=["uid"])
+            self.__dict__.update(self._original_data)
+
+    def delete(self):
+        """Delete this data source from HydroServer."""
+
+        if not self._uid:
+            raise AttributeError("This resource cannot be deleted: UID is not set.")
+
+        self._connection.workspaces.delete_api_key(
+            uid=self._workspace_id, api_key_id=self.uid
+        )
+        self._uid = None
+
+    def regenerate(self):
+        """Regenerates this API key. WARNING: Previous key will be invalidated."""
+
+        _, key = self._connection.workspaces.regenerate_api_key(
+            uid=self._workspace_id, api_key_id=self.uid
+        )
+
+        return key

hydroserverpy/api/models/iam/workspace.py

@@ -1,5 +1,6 @@
 from typing import List, Union, Optional, TYPE_CHECKING
 from uuid import UUID
+from datetime import datetime
 from pydantic import BaseModel, Field, EmailStr
 from ..base import HydroServerModel
 
@@ -8,6 +9,7 @@ if TYPE_CHECKING:
     from hydroserverpy.api.models import (
         Role,
         Collaborator,
+        APIKey,
         Account,
         Thing,
         ObservedProperty,
@@ -189,6 +191,79 @@ class Workspace(HydroServerModel, WorkspaceFields):
 
         super()._delete()
 
+    def list_api_keys(self) -> List["APIKey"]:
+        """Get all API keys associated with this workspace."""
+
+        return self._connection.workspaces.list_api_keys(uid=self.uid)
+
+    def get_api_key(self, api_key: Union["APIKey", UUID, str]) -> "APIKey":
+        """Get an API key associated with this workspace."""
+
+        return self._connection.workspaces.get_api_key(
+            uid=self.uid,
+            api_key_id=str(getattr(api_key, "uid", api_key))
+        )
+
+    def create_api_key(
+        self,
+        role: Union["Role", UUID, str],
+        name: str,
+        description: Optional[str] = None,
+        is_active: bool = True,
+        expires_at: Optional[datetime] = None
+    ):
+        """Create an API key associated with this workspace."""
+
+        api_key, key = self._connection.workspaces.create_api_key(
+            uid=self.uid,
+            role=role,
+            name=name,
+            description=description,
+            is_active=is_active,
+            expires_at=expires_at
+        )
+
+        return api_key, key
+
+    def update_api_key(
+            self,
+            api_key_id: Union[UUID, str],
+            role: Union["Role", UUID, str] = ...,
+            name: str = ...,
+            description: Optional[str] = ...,
+            is_active: bool = ...,
+            expires_at: Optional[datetime] = ...
+    ):
+        """Create an API key associated with this workspace."""
+
+        return self._connection.workspaces.update_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id,
+            role=role,
+            name=name,
+            description=description,
+            is_active=is_active,
+            expires_at=expires_at
+        )
+
+    def delete_api_key(self, api_key_id: Union[UUID, str]):
+        """Delete an API key associated with this workspace."""
+
+        return self._connection.workspaces.delete_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id
+        )
+
+    def regenerate_api_key(self, api_key_id: Union[UUID, str]):
+        """Regenerate an API key associated with this workspace."""
+
+        api_key, key = self._connection.workspaces.regenerate_api_key(
+            uid=self.uid,
+            api_key_id=api_key_id
+        )
+
+        return api_key, key
+
     def add_collaborator(
         self, email: EmailStr, role: Union["Role", UUID, str]
     ) -> "Collaborator":

hydroserverpy/api/services/iam/workspace.py

@@ -1,7 +1,8 @@
-from typing import TYPE_CHECKING, Union, List
+from typing import TYPE_CHECKING, Union, List, Tuple, Optional
 from pydantic import EmailStr
 from uuid import UUID
-from hydroserverpy.api.models import Workspace, Role, Collaborator
+from datetime import datetime
+from hydroserverpy.api.models import Workspace, Role, Collaborator, APIKey
 from ..base import EndpointService
 
 
@@ -107,6 +108,114 @@ class WorkspaceService(EndpointService):
         path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/collaborators"
         self._connection.request("delete", path, json={"email": email})
 
+    def list_api_keys(self, uid: Union[UUID, str]) -> List["APIKey"]:
+        """Get all API keys associated with a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys"
+        response = self._connection.request("get", path)
+
+        return [APIKey(_connection=self._connection, _uid=UUID(str(obj.pop("id"))), **obj) for obj in response.json()]
+
+    def get_api_key(self, uid: Union[UUID, str], api_key_id: Union[UUID, str]) -> "APIKey":
+        """Get an API key associated with a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{api_key_id}"
+        response = self._connection.request("get", path).json()
+
+        return APIKey(_connection=self._connection, _uid=UUID(str(response.pop("id"))), **response)
+
+    def create_api_key(
+        self,
+        uid: Union[UUID, str],
+        role: Union["Role", UUID, str],
+        name: str,
+        description: Optional[str] = None,
+        is_active: bool = True,
+        expires_at: Optional[datetime] = None
+    ) -> Tuple["APIKey", str]:
+        """Create an API key for a workspace."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys"
+        kwargs = {
+            "roleId": str(getattr(role, "uid", role)),
+            "name": name,
+            "description": description,
+            "isActive": is_active,
+            "expiresAt": expires_at
+        }
+        headers = {"Content-type": "application/json"}
+
+        response = self._connection.request(
+            "post", path, headers=headers, json=self._to_iso_time(kwargs)
+        ).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        ), response["key"]
+
+    def update_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str],
+        role: Union["Role", UUID, str] = ...,
+        name: str = ...,
+        description: Optional[str] = ...,
+        is_active: bool = ...,
+        expires_at: Optional[datetime] = ...
+    ) -> "APIKey":
+        """Update an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}"
+        kwargs = {
+            "roleId": ... if role is ... else str(getattr(role, "uid", role)),
+            "name": name,
+            "description": description,
+            "isActive": is_active,
+            "expiresAt": (
+                expires_at.isoformat()
+                if expires_at
+                not in (
+                    None,
+                    ...,
+                )
+                else None
+            )
+        }
+        headers = {"Content-type": "application/json"}
+
+        response = self._connection.request(
+            "patch", path, headers=headers,
+            json={k: v for k, v in kwargs.items() if v is not ...}
+        ).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        )
+
+    def delete_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str]
+    ):
+        """Delete an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}"
+        self._connection.request("delete", path)
+
+    def regenerate_api_key(
+        self,
+        uid: Union[UUID, str],
+        api_key_id: Union[UUID, str]
+    ):
+        """Regenerate an existing API key."""
+
+        path = f"/{self._api_route}/{self._endpoint_route}/{str(uid)}/api-keys/{str(api_key_id)}/regenerate"
+        response = self._connection.request("put", path).json()
+
+        return APIKey(
+            _connection=self._connection, _uid=UUID(str(response.pop("id"))), **response
+        ), response["key"]
+
     def transfer_ownership(self, uid: Union[UUID, str], email: str) -> None:
         """Transfer ownership of a workspace to another HydroServer user."""
 

hydroserverpy/etl/extractors/http_extractor.py

@@ -7,64 +7,79 @@ from .base import Extractor
 
 
 class HTTPExtractor(Extractor):
-    def __init__(
-        self,
-        url: str,
-        url_variables: dict = None,
-        params: dict = None,
-        headers: dict = None,
-        auth: tuple = None,
-    ):
-        self.url = self.format_url(url, url_variables or {})
-        self.params = params
-        self.headers = headers
-        self.auth = auth
-        self.start_date = None
+    def __init__(self, settings: object):
+        self.url = settings["urlTemplate"]
+        # self.url = self.format_url(url, url_variables or {})
+        # self.params = settings.get('params', )
+        # self.headers = headers
+        # self.auth = auth
 
     def prepare_params(self, data_requirements: Dict[str, TimeRange]):
-        start_times = [
-            req["start_time"] for req in data_requirements.values() if req["start_time"]
-        ]
+        pass
+        # TODO: Uncomment this once url templates work on in the Data Management App
+        # start_times = [
+        #     req["start_time"] for req in data_requirements.values() if req["start_time"]
+        # ]
 
-        if start_times:
-            oldest_start_time = min(start_times).isoformat()
-            start_time_key = self.params.pop("start_time_key", None)
-            if start_time_key:
-                self.params[start_time_key] = oldest_start_time
-                logging.info(
-                    f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
-                )
-            else:
-                logging.warning("'start_time_key' not found in params.")
+        # if start_times:
+        #     oldest_start_time = min(start_times)
+        #     start_time_key = self.params.pop("start_time_key", None)
+        #     if start_time_key:
+        #         self.params[start_time_key] = oldest_start_time
+        #         logging.info(
+        #             f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
+        #         )
+        #     else:
+        #         logging.warning("'start_time_key' not found in params.")
 
-        end_times = [
-            req["end_time"] for req in data_requirements.values() if req["end_time"]
-        ]
+        # end_times = [
+        #     req["end_time"] for req in data_requirements.values() if req["end_time"]
+        # ]
 
-        if end_times:
-            newest_end_time = max(end_times).isoformat()
-            end_time_key = self.params.pop("end_time_key", None)
-            if end_time_key:
-                self.params[end_time_key] = newest_end_time
-                logging.info(
-                    f"Set end_time to {newest_end_time} and removed 'end_time_key'"
-                )
-            else:
-                logging.warning("'end_time_key' not found in params.")
+        # if end_times:
+        #     newest_end_time = max(end_times)
+        #     end_time_key = self.params.pop("end_time_key", None)
+        #     if end_time_key:
+        #         self.params[end_time_key] = newest_end_time
+        #         logging.info(
+        #             f"Set end_time to {newest_end_time} and removed 'end_time_key'"
+        #         )
+        #     else:
+        #         logging.warning("'end_time_key' not found in params.")
 
     def extract(self):
         """
         Downloads the file from the HTTP/HTTPS server and returns a file-like object.
         """
-        response = requests.get(
-            url=self.url,
-            params=self.params,
-            headers=self.headers,
-            auth=self.auth,
-            stream=True,
-        )
-        response.raise_for_status()
-        logging.info(f"Successfully downloaded file from {response.url}")
+
+        logging.info(f"Requesting data from → {self.url}")
+
+        # endpoints = [
+        #     "https://httpbin.org/get",
+        #     "https://jsonplaceholder.typicode.com/posts/1",
+        #     "https://api.github.com",
+        #     "https://api.ipify.org?format=json",
+        #     "https://www.python.org/",
+        #     "https://waterservices.usgs.gov/nwis/iv/?&format=json&sites=01646500&parameterCd=00060",
+        #     "https://datahub.io/core/country-list/r/data.csv",
+        #     "https://raw.githubusercontent.com/cs109/2014_data/master/countries.csv",
+        #     # "https://rain-flow.slco.org/export/file/?delimiter=comma&site_id=68&data_start=2025-04-09&data_end=2025-05-09&device_id=2",
+        #     # "https://rain-flow.slco.org/export/file/?mime=txt&delimiter=comma&site_id=68&data_start=2025-05-09%2000:00:00&data_end=2025-05-09%2023:59:59&device_id=2"
+        # ]
+        # for url in endpoints:
+        #     try:
+        #         r = requests.get(url, timeout=10)
+        #         print(f"{url:50} → {r.status_code}")
+        #     except Exception as e:
+        #         print(f"{url:50} → ERROR: {e}")
+
+        try:
+            response = requests.get(self.url)
+        except Exception as e:
+            logging.error(f"Failed to fetch {repr(self.url)}: {e}")
+            raise
+
+        logging.info(f"Received response")
 
         data = BytesIO()
         for chunk in response.iter_content(chunk_size=8192):

hydroserverpy/etl/extractors/local_file_extractor.py

@@ -6,8 +6,12 @@ from ..types import TimeRange
 
 
 class LocalFileExtractor(Extractor):
-    def __init__(self, filepath: str):
-        self.filepath = filepath
+    def __init__(self, settings: object):
+        if "path" not in settings:
+            message = "Missing required setting 'path' in LocalFileExtractor settings."
+            logging.error(message)
+            raise ValueError(message)
+        self.path = settings["path"]
 
     def prepare_params(self, data_requirements: Dict[str, TimeRange]):
         pass
@@ -17,9 +21,9 @@ class LocalFileExtractor(Extractor):
         Opens the file and returns a file-like object.
         """
         try:
-            file_handle = open(self.filepath, "r")
-            logging.info(f"Successfully opened file '{self.filepath}'.")
+            file_handle = open(self.path, "r")
+            logging.info(f"Successfully opened file '{self.path}'.")
             return file_handle
         except Exception as e:
-            logging.error(f"Error opening file '{self.filepath}': {e}")
+            logging.error(f"Error opening file '{self.path}': {e}")
             return None

hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -1,5 +1,8 @@
+import datetime
 from hydroserverpy import HydroServer
 from typing import Dict, Optional
+
+from src.hydroserverpy.etl.types import TimeRange
 from .base import Loader
 import logging
 import pandas as pd
@@ -13,20 +16,25 @@ class HydroServerLoader(HydroServer, Loader):
     def __init__(
         self,
         host: str,
-        username: Optional[str] = None,
+        email: Optional[str] = None,
         password: Optional[str] = None,
         apikey: Optional[str] = None,
-        api_route: str = "api",
     ):
-        super().__init__(host, username, password, apikey, api_route)
+        super().__init__(
+            host=host,
+            email=email,
+            password=password,
+            apikey=apikey,
+        )
 
-    def load(self, data: pd.DataFrame, source_target_map) -> None:
+    def load(self, data: pd.DataFrame, payload_settings) -> None:
         """
         Load observations from a DataFrame to the HydroServer.
 
         :param data: A Pandas DataFrame where each column corresponds to a datastream.
         """
-        data_requirements = self.get_data_requirements(source_target_map)
+        mappings = payload_settings["mappings"]
+        time_ranges = self.get_data_requirements(mappings)
         for ds_id in data.columns:
             if ds_id == "timestamp":
                 continue
@@ -35,9 +43,17 @@ class HydroServerLoader(HydroServer, Loader):
             df.rename(columns={ds_id: "value"}, inplace=True)
             df.dropna(subset=["value"], inplace=True)
 
-            phenomenon_end_time = data_requirements[ds_id]["start_time"]
-            if phenomenon_end_time:
-                df = df[df["timestamp"] > phenomenon_end_time]
+            # ensure the timestamp column is UTC‑aware
+            timestamp_column = df["timestamp"]
+            if timestamp_column.dt.tz is None:
+                df["timestamp"] = timestamp_column.dt.tz_localize("UTC")
+
+            time_range = time_ranges[ds_id]
+            start_ts = pd.to_datetime(time_range["start_time"], utc=True)
+
+            if start_ts:
+                df = df[df["timestamp"] > start_ts]
+            logging.info(f"start cutoff for data loading {start_ts}")
             if df.empty:
                 logging.warning(
                     f"No new data to upload for datastream {ds_id}. Skipping."
@@ -45,24 +61,31 @@ class HydroServerLoader(HydroServer, Loader):
                 continue
             self.datastreams.load_observations(uid=ds_id, observations=df)
 
-    def get_data_requirements(
-        self, source_target_map
-    ) -> Dict[str, Dict[str, pd.Timestamp]]:
+    def get_data_requirements(self, source_target_map) -> Dict[str, TimeRange]:
         """
         Each target system needs to be able to answer the question: 'What data do you need?'
         and return a time range for each target time series. Usually the answer will be
         'anything newer than my most recent observation'.
         """
         data_requirements = {}
-        for ds_id in source_target_map.values():
-            datastream = self.datastreams.get(uid=ds_id)
+        target_ids = [mapping["targetIdentifier"] for mapping in source_target_map]
+        for id in target_ids:
+            datastream = self.datastreams.get(uid=id)
             if not datastream:
                 message = "Couldn't fetch target datastream. ETL process aborted."
                 logging.error(message)
                 raise message
-            start_time = pd.Timestamp(
+
+            start_ts = pd.Timestamp(
                 datastream.phenomenon_end_time or "1970-01-01T00:00:00Z"
             )
-            end_time = pd.Timestamp.now()
-            data_requirements[ds_id] = {"start_time": start_time, "end_time": end_time}
+            if start_ts.tzinfo is None:
+                start_ts = start_ts.tz_localize("UTC")
+
+            end_ts = pd.Timestamp.now(tz="UTC")
+
+            data_requirements[id] = {
+                "start_time": start_ts.isoformat(),
+                "end_time": end_ts.isoformat(),
+            }
         return data_requirements

hydroserverpy/etl/transformers/base.py

@@ -1,9 +1,22 @@
 from abc import ABC, abstractmethod
+from datetime import timedelta, timezone
 import logging
+from typing import Union
 import pandas as pd
 
 
 class Transformer(ABC):
+    def __init__(self, settings: object):
+        # timestampFormat will be the strs: 'utc', 'ISO8601', 'constant', or some custom openStrftime.
+        # If 'constant', then the system will append the timestamp_offset to the end of it.
+        self.timestamp_format = settings.get("timestampFormat", "ISO8601")
+        self.timestamp_offset: str = settings.get("timestampOffset", "+0000")
+        self.timestamp_key: Union[str, int] = settings["timestampKey"]
+
+        if isinstance(self.timestamp_key, int):
+            # Users will always interact in 1-based, so if the key is a column index, convert to 0-based
+            self.timestamp_key = self.timestamp_key - 1
+
     @abstractmethod
     def transform(self, *args, **kwargs) -> None:
         pass
@@ -12,41 +25,93 @@ class Transformer(ABC):
     def needs_datastreams(self) -> bool:
         return False
 
-    @staticmethod
-    def standardize_dataframe(
-        df,
-        datastream_ids,
-        timestamp_column: str = "timestamp",
-        timestamp_format: str = "ISO8601",
-    ):
+    def standardize_dataframe(self, df, payload_mappings):
+        rename_map = {
+            mapping["sourceIdentifier"]: mapping["targetIdentifier"]
+            for mapping in payload_mappings
+        }
+
         df.rename(
-            columns={timestamp_column: "timestamp", **datastream_ids},
+            columns={self.timestamp_key: "timestamp", **rename_map},
             inplace=True,
         )
 
         # Verify timestamp column is present in the DataFrame
         if "timestamp" not in df.columns:
-            message = f"Timestamp column '{timestamp_column}' not found in data."
+            message = f"Timestamp column '{self.timestamp_key}' not found in data."
             logging.error(message)
             raise ValueError(message)
 
-        # Verify that all datastream_ids are present in the DataFrame
-        expected_columns = set(datastream_ids.values())
-        actual_columns = set(df.columns)
-        missing_datastream_ids = expected_columns - actual_columns
-
-        if missing_datastream_ids:
+        # verify datastream columns
+        expected = set(rename_map.values())
+        missing = expected - set(df.columns)
+        if missing:
             raise ValueError(
                 "The following datastream IDs are specified in the config file but their related keys could not be "
-                f"found in the source system's extracted data: {missing_datastream_ids}"
+                f"found in the source system's extracted data: {missing}"
             )
 
-        # Keep only 'timestamp' and datastream_id columns
-        columns_to_keep = ["timestamp"] + list(expected_columns)
-        df = df[columns_to_keep]
+        # keep only timestamp + datastream columns; remove the rest inplace
+        to_keep = ["timestamp", *expected]
+        df.drop(columns=df.columns.difference(to_keep), inplace=True)
+
+        df["timestamp"] = self._parse_timestamps(df["timestamp"])
 
-        # Convert timestamp column to datetime if not already
-        if not pd.api.types.is_datetime64_any_dtype(df["timestamp"]):
-            df["timestamp"] = pd.to_datetime(df["timestamp"], format=timestamp_format)
+        df.drop_duplicates(subset=["timestamp"], keep="last")
+        logging.info(f"standardized dataframe created: {df.shape}")
+        logging.info(f"{df.info()}")
+        logging.info(f"{df.head()}")
 
         return df
+
+    def _parse_timestamps(self, raw_series: pd.Series) -> pd.Series:
+        """Return a Series of pandas UTC datetimes for the four supported modes."""
+        logging.info(f"parsing timestamps. Format: {self.timestamp_format}")
+
+        fmt = self.timestamp_format.lower()
+
+        VALID_KEYS = {"utc", "iso8601", "constant"}
+        if fmt not in VALID_KEYS and "%" not in self.timestamp_format:
+            raise ValueError(
+                f"timestamp_format must be one of {', '.join(VALID_KEYS)} "
+                "or a valid strftime pattern."
+            )
+
+        series = raw_series.str.strip()
+
+        if fmt == "utc":
+            # Accept Z-suffix, no offset, fractional seconds, etc.
+            parsed = pd.to_datetime(series, utc=True, errors="coerce")
+
+        elif fmt == "iso8601":
+            # pandas reads the embedded offset, then we shift to UTC
+            parsed = pd.to_datetime(series, errors="coerce").dt.tz_convert("UTC")
+
+        elif fmt == "constant":
+            offset = str(self.timestamp_offset).strip()
+            if not (len(offset) == 5 and offset[0] in "+-"):
+                raise ValueError(f"Invalid timestampOffset: {self.timestamp_offset}")
+
+            sign_multiplier = 1 if offset[0] == "+" else -1
+            hours = int(offset[1:3])
+            minutes = int(offset[3:5])
+            total_minutes = sign_multiplier * (hours * 60 + minutes)
+            local_timezone = timezone(timedelta(minutes=total_minutes))
+
+            naive_times = pd.to_datetime(series, errors="coerce")
+            localized_times = naive_times.dt.tz_localize(local_timezone)
+            parsed = localized_times.dt.tz_convert("UTC")
+
+        else:
+            logging.info(f"timestamp format is custom {self.timestamp_format}")
+            parsed = pd.to_datetime(
+                series, format=self.timestamp_format, errors="coerce"
+            ).dt.tz_localize("UTC")
+
+        if parsed.isna().any():
+            bad_rows = series[parsed.isna()].head(5).tolist()
+            logging.warning(
+                f"{parsed.isna().sum()} timestamps failed to parse. Sample bad values: {bad_rows}"
+            )
+
+        return parsed

hydroserverpy/etl/transformers/csv_transformer.py

@@ -1,28 +1,25 @@
+from io import StringIO
 import logging
 import pandas as pd
-from typing import Dict, Optional, Union
+from typing import Iterable, Union
 from .base import Transformer
 
 
 class CSVTransformer(Transformer):
-    def __init__(
-        self,
-        header_row: Optional[int],
-        data_start_row: int,
-        timestamp_column: Union[str, int],
-        datastream_ids: Dict[Union[str, int], str],
-        delimiter: Optional[str] = ",",
-        timestamp_format: Optional[str] = "ISO8601",
-    ):
+    def __init__(self, settings: object):
+        super().__init__(settings)
+
         # Pandas is zero-based while CSV is one-based so convert
-        self.header_row = None if header_row is None else header_row - 1
-        self.data_start_row = data_start_row - 1
-        self.timestamp_column = self.convert_to_zero_based(timestamp_column)
-        self.datastream_ids = datastream_ids
-        self.timestamp_format = timestamp_format
-        self.delimiter = delimiter
+        self.header_row = (
+            None if settings.get("headerRow") is None else settings["headerRow"] - 1
+        )
+        self.data_start_row = (
+            settings["dataStartRow"] - 1 if "dataStartRow" in settings else 0
+        )
+        self.delimiter = settings.get("delimiter", ",")
+        self.identifier_type = settings.get("identifierType", "name")
 
-    def transform(self, data_file) -> Union[pd.DataFrame, None]:
+    def transform(self, data_file, mappings) -> Union[pd.DataFrame, None]:
         """
         Transforms a CSV file-like object into a Pandas DataFrame where the column
         names are replaced with their target datastream ids.
@@ -33,16 +30,22 @@ class CSVTransformer(Transformer):
             observations_map (dict): Dict mapping datastream IDs to pandas DataFrames.
         """
 
+        clean_file = self._strip_comments(data_file)
+        source_identifiers = [mapping["sourceIdentifier"] for mapping in mappings]
+
         try:
+            # Pandas’ heuristics strip offsets and silently coerce failures to strings.
+            # Reading as pure text guarantees we always start with exactly what was in the file.
+            # Timestamps will be parsed at df standardization time.
             df = pd.read_csv(
-                data_file,
-                delimiter=self.delimiter,
+                clean_file,
+                sep=self.delimiter,
                 header=self.header_row,
-                parse_dates=[self.timestamp_column],
-                date_format=self.timestamp_format,
-                skiprows=self.calculate_skiprows(),
-                usecols=[self.timestamp_column] + list(self.datastream_ids.keys()),
+                skiprows=self._build_skiprows(),
+                usecols=[self.timestamp_key] + source_identifiers,
+                dtype={self.timestamp_key: "string"},
             )
+            logging.info(f"CSV file read into dataframe: {df.shape}")
         except Exception as e:
             logging.error(f"Error reading CSV data: {e}")
             return None
@@ -50,39 +53,25 @@ class CSVTransformer(Transformer):
         if self.header_row is None:
             df.columns = list(range(1, len(df.columns) + 1))
 
-        return self.standardize_dataframe(
-            df, self.datastream_ids, self.timestamp_column, self.timestamp_format
-        )
+        return self.standardize_dataframe(df, mappings)
 
-    def calculate_skiprows(self):
+    def _strip_comments(self, stream: Iterable[Union[str, bytes]]) -> StringIO:
         """
-        Calculates the skiprows parameter for pd.read_csv.
-
-        Returns:
-            skiprows (list or None): List of row indices to skip, or None if no rows need to be skipped.
-        Raises:
-            ValueError: If header_row is not compatible with data_start_row.
+        Remove lines whose first non-blank char is '#'.
+        Works for both text and binary iterables.
         """
-        if self.data_start_row == 0:
-            if self.header_row is not None:
-                # Cannot have a header row if data starts at the first row
-                raise ValueError(
-                    "header_row must be None when data_start_row is 1 (first row)"
-                )
-            return None  # No rows to skip
+        clean: list[str] = []
 
-        skiprows = list(range(self.data_start_row))
+        for raw in stream:
+            # normalize to bytes
+            b = raw if isinstance(raw, bytes) else raw.encode("utf-8", "ignore")
+            if b.lstrip().startswith(b"#"):
+                continue
+            clean.append(
+                raw.decode("utf-8", "ignore") if isinstance(raw, bytes) else raw
+            )
 
-        if self.header_row is not None:
-            if self.header_row >= self.data_start_row:
-                raise ValueError("header_row must be less than data_start_row")
-            if self.header_row in skiprows:
-                # Do not skip the header row
-                skiprows.remove(self.header_row)
-        return skiprows
+        return StringIO("".join(clean))
 
-    @staticmethod
-    def convert_to_zero_based(index: Union[str, int]) -> Union[str, int]:
-        if isinstance(index, int):
-            return index - 1
-        return index
+    def _build_skiprows(self):
+        return lambda idx: idx != self.header_row and idx < self.data_start_row

hydroserverpy/etl/transformers/json_transformer.py

@@ -7,27 +7,11 @@ import jmespath
 
 
 class JSONTransformer(Transformer):
-    def __init__(
-        self,
-        query_string: str,
-        datastream_ids: Dict[str, str],
-        timestamp_format: Optional[str] = "ISO8601",
-    ):
-        """
-        Initializes the JSONTransformer.
-
-        Parameters:
-            query_string (str): JMESPath to the data array containing time series data.
-            Since JMESPath can natively rename column names, the assumption is the timestamp column
-            is always named 'timestamp' or converted to 'timestamp' in the JMESPath query.
-            datastream_ids (dict): Mapping from JSON field names to datastream IDs.
-            timestamp_format (str, optional): The format of the timestamp, if it needs special parsing.
-        """
-        self.query_string = query_string
-        self.datastream_ids = datastream_ids
-        self.timestamp_format = timestamp_format
+    def __init__(self, settings: object):
+        super().__init__(settings)
+        self.JMESPath = settings["JMESPath"]
 
-    def transform(self, data_file):
+    def transform(self, data_file, mappings):
         """
         Transforms a JSON file-like object into the standard Pandas dataframe format.
         Since JMESPath can natively rename column names, the assumption is the timestamp column
@@ -47,15 +31,11 @@ class JSONTransformer(Transformer):
 
         df = pd.DataFrame(data_points)
 
-        return self.standardize_dataframe(
-            df,
-            self.datastream_ids,
-            timestamp_format=self.timestamp_format,
-        )
+        return self.standardize_dataframe(df, mappings)
 
     def extract_data_points(self, json_data: Any) -> Optional[List[dict]]:
         """Extracts data points from the JSON data using the data_path."""
-        data_points = jmespath.search(self.query_string, json_data)
+        data_points = jmespath.search(self.JMESPath, json_data)
 
         if isinstance(data_points, dict):
             data_points = [data_points]

{hydroserverpy-1.0.0.dist-info → hydroserverpy-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.0.0
+Version: 1.1.0
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.0.0.dist-info → hydroserverpy-1.1.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 hydroserverpy/__init__.py,sha256=FgaGFyhCjwmpJYEKNzOZxvfRx2neWMaOybj1z02_VSE,218
 hydroserverpy/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/http.py,sha256=C5DgvEeiu54RaL-9oouFPSKosC8Uy5Qdwm5hYh2Ps-s,620
-hydroserverpy/api/main.py,sha256=OWJpCIuheBgWQA7R33BiG89-upMS-a3K_AScbVeVjxQ,4760
-hydroserverpy/api/models/__init__.py,sha256=ELrf3b7Aix7YcVF__Q_8e_G_FF8GYlX0J5l7fkGcHnY,690
+hydroserverpy/api/main.py,sha256=r-KD1Xyufzyt4YvRRhZbAngrj5h9yWLXiznx1Yda_8E,4766
+hydroserverpy/api/models/__init__.py,sha256=buOhJ2Bf9yI0GftSyulpR74A5IhyyKJrXY1xANicohw,744
 hydroserverpy/api/models/base.py,sha256=dc2tfMSgizymxAAOVURfy7Jzeh6xIiiq7hfWZI7l1_Q,2280
 hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/etl/data_archive.py,sha256=u-gpvUsaWaw0kyF3bPMm2e55Jx2yhvSV9ufXXaNtrTc,3429
@@ -11,9 +11,10 @@ hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJF
 hydroserverpy/api/models/etl/orchestration_system.py,sha256=25En2G0z1gQzN-RW3UlrEGgkC952QDW21oYnawCX8hY,2357
 hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
+hydroserverpy/api/models/iam/apikey.py,sha256=FLppIWqoBlFYqumecF1cKVm4R2-ZacyKrpObMW1g50s,2465
 hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F5eGDKaSHmH62Q8Q,1007
 hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
-hydroserverpy/api/models/iam/workspace.py,sha256=s9u1oZyOdxM7txjJARFcIBrWMHQSDxODdreiatFsXJs,7331
+hydroserverpy/api/models/iam/workspace.py,sha256=rlRL52fAQrCZqmqMnCUNvBIWrlUGoYtQd16zCg-esWU,9630
 hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/sta/datastream.py,sha256=vRjgwAKaoBJEtgUXrZjIS-VuIZsCilm7FRwbvLS8Y8o,11186
 hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
@@ -29,7 +30,7 @@ hydroserverpy/api/services/etl/data_archive.py,sha256=hlNJOHJSZ1kV2n2xivWIBtT1Eo
 hydroserverpy/api/services/etl/data_source.py,sha256=DCgTyh8lF2iwh4uszePFg9UupXxJCN7Ww9Ut1MQKHis,6491
 hydroserverpy/api/services/etl/orchestration_system.py,sha256=JFuSJJUq4JJUt8KlZ-Ga0ktyQIe2U0Sa7ogd4oLjex4,2166
 hydroserverpy/api/services/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/api/services/iam/workspace.py,sha256=jJiqkMxFEp9REjR4LXyVp2o45CGBrrEadGelPPCuRJs,4547
+hydroserverpy/api/services/iam/workspace.py,sha256=KQ1zH7EnEcBrpeF-2APAo3aMiiSGCaUgOy_GwRQtyRI,8395
 hydroserverpy/api/services/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/services/sta/datastream.py,sha256=_m-xFom3z8wo5-1_q8NjWUpcw36wYv1brIG7xeGGadk,12402
 hydroserverpy/api/services/sta/observed_property.py,sha256=nRlqBldJpXlj8VOZ4EwNOs4ZgmBw5w-EqAChfM3Z0Z0,2908
@@ -44,23 +45,23 @@ hydroserverpy/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,13
 hydroserverpy/etl/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/etl/extractors/base.py,sha256=GZKJfAhfJedRcNagnoqUiDZn286r-JzM7dW_F1dWsfY,275
 hydroserverpy/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
-hydroserverpy/etl/extractors/http_extractor.py,sha256=-duQwnsFBk4NQS2qhO55evcCUOnrBe3JX_LU9RyysX4,2709
-hydroserverpy/etl/extractors/local_file_extractor.py,sha256=T_Y9NTO0cC5L9mDPbIG6wYlXDQoatg8MobP97liFl4U,692
+hydroserverpy/etl/extractors/http_extractor.py,sha256=FYC0nzb8Yt0UvLqikQaBD4KjkWkMLS1EYpzwiS1u3JE,3807
+hydroserverpy/etl/extractors/local_file_extractor.py,sha256=UQ37t_NLqkZQ7biPoOrShU4NsV1OJEtg-U1mYIQGrPk,883
 hydroserverpy/etl/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/etl/loaders/base.py,sha256=DrA9u7SNBxkPKqaszlP368yNbxihdqIGzP8rA6NAp6U,295
-hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=bl4Z5TkXgJyKEHuPdY7LSrKMKOoZW_EDFl8dIRwDkv4,2549
+hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=ofXgnD0B8KPD0G6MW0AwTQYpGm9Q11YJmHIhdw354jg,3204
 hydroserverpy/etl/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/etl/transformers/base.py,sha256=237oVBhS3HQ3fcE4bZT5U1437WzV2x5kOFC229DY53M,1741
-hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3-7LxwSSeuQdou0iiCjdM,3238
-hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
+hydroserverpy/etl/transformers/base.py,sha256=OsLyjtLwPOCGdeoH2Ui0d0GqD2n55-9ch17UjZRFU7I,4615
+hydroserverpy/etl/transformers/csv_transformer.py,sha256=0kWfRKPwiGxCNZ87Q4SiBlfM3PuKL6upc1ljphBY89o,2891
+hydroserverpy/etl/transformers/json_transformer.py,sha256=R7tSyDB4Wn1snP75ctbEDMaMCdjyhPnMzN_W2VV3Mv4,1506
 hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
 hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=0ueBphEaAAlsb0cn71Ihgd5zOD8Zdu4Ts_yGwvXW53M,14544
 hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
 hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
-hydroserverpy-1.0.0.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
-hydroserverpy-1.0.0.dist-info/METADATA,sha256=Da0zrvryIMT9-eYhmsNES05Ebu7SwqYQIAs0qIe_19c,530
-hydroserverpy-1.0.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-hydroserverpy-1.0.0.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
-hydroserverpy-1.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-hydroserverpy-1.0.0.dist-info/RECORD,,
+hydroserverpy-1.1.0.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
+hydroserverpy-1.1.0.dist-info/METADATA,sha256=KzHUpDAMNBWMZXZAipD4yDZUQhPrBgicgW1qvhqNsBE,530
+hydroserverpy-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hydroserverpy-1.1.0.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
+hydroserverpy-1.1.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+hydroserverpy-1.1.0.dist-info/RECORD,,

{hydroserverpy-1.0.0.dist-info → hydroserverpy-1.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any