hydroserverpy 1.5.1__py3-none-any.whl

hydroserverpy/api/models/etl/timestamp_parser.py

@@ -0,0 +1,112 @@
+from functools import cached_property
+import logging
+from datetime import datetime, timedelta, timezone
+import re
+from typing import Literal, Optional, Union, get_args
+from zoneinfo import ZoneInfo
+import pandas as pd
+from pydantic import BaseModel, Field
+
+TimestampFormat = Literal["ISO8601", "naive", "custom"]
+ALLOWED_TIMESTAMP_FORMATS = {m.lower() for m in get_args(TimestampFormat)}
+TimezoneMode = Literal["utc", "daylightSavings", "fixedOffset", "embeddedOffset"]
+ALLOWED_TIMEZONE_MODES = {m.lower() for m in get_args(TimezoneMode)}
+
+
+class Timestamp(BaseModel):
+    format: TimestampFormat
+    timezone_mode: TimezoneMode = Field("embeddedOffset", alias="timezoneMode")
+    custom_format: Optional[str] = Field(None, alias="customFormat")
+    timezone: Optional[str] = None
+    key: Optional[str] = None
+
+
+class TimestampParser:
+    def __init__(self, raw: Union[Timestamp, dict]):
+        if isinstance(raw, dict):
+            self.timestamp = Timestamp.model_validate(raw)
+        else:
+            self.timestamp = raw
+
+        if self.timestamp.format.lower() not in ALLOWED_TIMESTAMP_FORMATS:
+            raise ValueError(
+                f"timestamp format {self.timestamp.format!r} must be one of {ALLOWED_TIMESTAMP_FORMATS}"
+            )
+
+        self.tz_mode = self.timestamp.timezone_mode.lower()
+        if self.tz_mode not in ALLOWED_TIMEZONE_MODES and "%" not in self.tz_mode:
+            raise ValueError(
+                f"timezone mode {self.tz_mode} must be one of {', '.join(ALLOWED_TIMEZONE_MODES)} "
+            )
+
+    @cached_property
+    def tz(self):
+        if self.tz_mode == "fixedoffset":
+            offset = self.timestamp.timezone.strip()
+            if len(offset) != 5 or offset[0] not in "+-":
+                raise ValueError(f"Invalid timezone: {offset}")
+            sign = 1 if offset[0] == "+" else -1
+            hrs, mins = int(offset[1:3]), int(offset[3:5])
+            return timezone(timedelta(minutes=sign * (hrs * 60 + mins)))
+        if self.tz_mode == "daylightsavings":
+            return ZoneInfo(self.timestamp.timezone)
+        if self.tz_mode == "utc":
+            return timezone.utc
+
+    def _convert_series_to_UTC(self, s: pd.Series):
+        timestamp_fmt = self.timestamp.format.lower()
+
+        if timestamp_fmt == "iso8601":
+            return pd.to_datetime(s, utc=True, errors="coerce")
+
+        if timestamp_fmt == "custom":
+            pattern = self.timestamp.custom_format or ""
+            naive = pd.to_datetime(s, format=pattern, errors="coerce")
+        else:
+            naive = pd.to_datetime(s, errors="coerce")
+
+        tz_mode = self.timestamp.timezone_mode.lower()
+        if tz_mode == "utc":
+            return pd.to_datetime(naive, utc=True, errors="coerce")
+
+        localized = naive.dt.tz_localize(self.tz)
+        return localized.dt.tz_convert(timezone.utc)
+
+    def parse_series(self, raw_series: pd.Series) -> pd.Series:
+        if pd.api.types.is_datetime64_any_dtype(raw_series):
+            s = raw_series  # already datetimes
+        else:
+            s = raw_series.astype("string", copy=False).str.strip()
+        parsed = self._convert_series_to_UTC(s)
+
+        if parsed.isna().any():
+            bad_rows = s[parsed.isna()].head(2).tolist()
+            logging.warning(
+                f"{parsed.isna().sum()} timestamps failed to parse. "
+                f"Sample bad values: {bad_rows}"
+            )
+
+        return parsed
+
+    def utc_to_string(self, dt: Union[datetime, pd.Timestamp]) -> str:
+        """
+        Convert a UTC datetime or pd.Timestamp to a custom string format.
+
+        Some external APIs are picky about their timestamp formats, so we need the ability to pull a
+        UTC timestamp from HydroServer and format it into a custom string.
+        """
+        if isinstance(dt, pd.Timestamp):
+            dt = dt.to_pydatetime()
+
+        tz_format = self.timestamp.format.lower()
+        if tz_format == "iso8601":
+            return dt.astimezone(timezone.utc).isoformat()
+
+        if tz_format == "naive":
+            return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
+
+        if tz_format == "custom":
+            logging.info(f"custom timestamp: ... {self.timestamp}")
+            return dt.astimezone(self.tz).strftime(self.timestamp.custom_format)
+
+        raise ValueError(f"Unknown timestamp.format: {self.timestamp.format!r}")

hydroserverpy/api/models/etl/transformers/__init__.py

@@ -0,0 +1,5 @@
+from .base import Transformer
+from .json_transformer import JSONTransformer
+from .csv_transformer import CSVTransformer
+
+__all__ = ["Transformer", "JSONTransformer", "CSVTransformer"]

hydroserverpy/api/models/etl/transformers/base.py

@@ -0,0 +1,135 @@
+from abc import ABC, abstractmethod
+import ast
+from functools import lru_cache
+import logging
+import re
+from typing import List, Union
+import pandas as pd
+
+from ..timestamp_parser import TimestampParser
+from ..etl_configuration import MappingPath, TransformerConfig, SourceTargetMapping
+
+ALLOWED_AST = (
+    ast.Expression,
+    ast.BinOp,
+    ast.UnaryOp,
+    ast.Add,
+    ast.Sub,
+    ast.Mult,
+    ast.Div,
+    ast.UAdd,
+    ast.USub,
+    ast.Name,
+    ast.Load,
+    ast.Constant,
+)
+
+
+def _canonicalize_expr(expr: str) -> str:
+    # normalize whitespace for cache hits; parentheses remain intact
+    return re.sub(r"\s+", "", expr)
+
+
+@lru_cache(maxsize=256)
+def _compile_arithmetic_expr_canon(expr_no_ws: str):
+    tree = ast.parse(expr_no_ws, mode="eval")
+    for node in ast.walk(tree):
+        if not isinstance(node, ALLOWED_AST):
+            raise ValueError(
+                "Only +, -, *, / with 'x' and numeric literals are allowed."
+            )
+        if isinstance(node, ast.Name) and node.id != "x":
+            raise ValueError("Only the variable 'x' is allowed.")
+        if isinstance(node, ast.Constant):
+            val = node.value
+            if isinstance(val, bool) or not isinstance(val, (int, float)):
+                raise ValueError("Only numeric literals are allowed.")
+    return compile(tree, "<expr>", "eval")
+
+
+def _compile_arithmetic_expr(expr: str):
+    return _compile_arithmetic_expr_canon(_canonicalize_expr(expr))
+
+
+class Transformer(ABC):
+    def __init__(self, transformer_config: TransformerConfig):
+        self.cfg = transformer_config
+        self.timestamp = transformer_config.timestamp
+        self.timestamp_parser = TimestampParser(self.timestamp)
+
+    @abstractmethod
+    def transform(self, *args, **kwargs) -> None:
+        pass
+
+    @property
+    def needs_datastreams(self) -> bool:
+        return False
+
+    def standardize_dataframe(
+        self, df: pd.DataFrame, mappings: List[SourceTargetMapping]
+    ):
+        if not df.empty:
+            logging.info(f"Read payload into dataframe: {df.iloc[0].to_dict()}")
+        else:
+            logging.info("Read payload into dataframe: [empty dataframe]")
+
+        # 1) Normalize timestamp column
+        df.rename(columns={self.timestamp.key: "timestamp"}, inplace=True)
+        if "timestamp" not in df.columns:
+            msg = f"Timestamp column '{self.timestamp.key}' not found in data."
+            logging.error(msg)
+            raise ValueError(msg)
+        logging.info(f"Renamed timestamp column to 'timestamp'")
+
+        df["timestamp"] = self.timestamp_parser.parse_series(df["timestamp"])
+        df = df.drop_duplicates(subset=["timestamp"], keep="last")
+
+        def _resolve_source_col(s_id: Union[str, int]) -> str:
+            if isinstance(s_id, int) and s_id not in df.columns:
+                try:
+                    return df.columns[s_id]
+                except IndexError:
+                    raise ValueError(
+                        f"Source index {s_id} is out of range for extracted data."
+                    )
+            if s_id not in df.columns:
+                raise ValueError(f"Source column '{s_id}' not found in extracted data.")
+            return s_id
+
+        def _apply_transformations(series: pd.Series, path: MappingPath) -> pd.Series:
+            out = series  # accumulator for sequential transforms
+            if out.dtype == "object":
+                out = pd.to_numeric(out, errors="coerce")
+
+            for transformation in path.data_transformations:
+                if transformation.type == "expression":
+                    code = _compile_arithmetic_expr(transformation.expression)
+                    try:
+                        out = eval(code, {"__builtins__": {}}, {"x": out})
+                    except Exception as ee:
+                        logging.exception(
+                            "Data transformation failed for expression=%r",
+                            transformation.expression,
+                        )
+                        raise
+                else:
+                    msg = f"Unsupported transformation type: {transformation.type}"
+                    logging.error(msg)
+                    raise ValueError(msg)
+            return out
+
+        # source target mappings may be one to many. Therefore, create a new column for each target and apply transformations
+        transformed_df = pd.DataFrame(index=df.index)
+        for m in mappings:
+            src_col = _resolve_source_col(m.source_identifier)
+            base = df[src_col]
+            for path in m.paths:
+                target_col = str(path.target_identifier)
+                transformed_df[target_col] = _apply_transformations(base, path)
+
+        # 6) Keep only timestamp + target columns
+        df = pd.concat([df[["timestamp"]], pd.DataFrame(transformed_df)], axis=1)
+
+        logging.info(f"standardized dataframe created: {df.shape}")
+
+        return df

hydroserverpy/api/models/etl/transformers/csv_transformer.py

@@ -0,0 +1,88 @@
+from io import StringIO
+import logging
+import pandas as pd
+from typing import Iterable, List, Union
+from .base import Transformer
+from ..etl_configuration import TransformerConfig, SourceTargetMapping
+
+
+class CSVTransformer(Transformer):
+    def __init__(self, transformer_config: TransformerConfig):
+        super().__init__(transformer_config)
+
+        # Pandas is zero-based while CSV is one-based so convert
+        self.header_row = (
+            None if self.cfg.header_row is None else self.cfg.header_row - 1
+        )
+        self.data_start_row = (
+            self.cfg.data_start_row - 1 if self.cfg.data_start_row else 0
+        )
+        self.delimiter = self.cfg.delimiter or ","
+        self.identifier_type = self.cfg.identifier_type or "name"
+
+    def transform(
+        self, data_file, mappings: List[SourceTargetMapping]
+    ) -> Union[pd.DataFrame, None]:
+        """
+        Transforms a CSV file-like object into a Pandas DataFrame where the column
+        names are replaced with their target datastream ids.
+
+        Parameters:
+            data_file: File-like object containing CSV data.
+        Returns:
+            observations_map (dict): Dict mapping datastream IDs to pandas DataFrames.
+        """
+
+        clean_file = self._strip_comments(data_file)
+        use_index = self.identifier_type == "index"
+
+        if use_index:
+            # Users will always interact in 1-based, so if the key is a column index, convert to 0-based to work with Pandas
+            timestamp_pos = int(self.timestamp.key) - 1
+            usecols = [timestamp_pos] + [int(m.source_identifier) - 1 for m in mappings]
+        else:
+            usecols = [self.timestamp.key] + [m.source_identifier for m in mappings]
+
+        try:
+            # Pandas’ heuristics strip offsets and silently coerce failures to strings.
+            # Reading as pure text guarantees we always start with exactly what was in the file.
+            # Timestamps will be parsed at df standardization time.
+            df = pd.read_csv(
+                clean_file,
+                sep=self.delimiter,
+                header=0,
+                skiprows=self._build_skiprows(),
+                usecols=usecols,
+                dtype={self.timestamp.key: "string"},
+            )
+            logging.info(f"CSV file read into dataframe: {df.shape}")
+        except Exception as e:
+            logging.error(f"Error reading CSV data: {e}")
+            return None
+
+        # In index mode, relabel columns back to original 1-based indices so base transformer can use integer labels directly
+        if use_index:
+            df.columns = [(c + 1) if isinstance(c, int) else c for c in usecols]
+
+        return self.standardize_dataframe(df, mappings)
+
+    def _strip_comments(self, stream: Iterable[Union[str, bytes]]) -> StringIO:
+        """
+        Remove lines whose first non-blank char is '#'.
+        Works for both text and binary iterables.
+        """
+        clean: list[str] = []
+
+        for raw in stream:
+            # normalize to bytes
+            b = raw if isinstance(raw, bytes) else raw.encode("utf-8", "ignore")
+            if b.lstrip().startswith(b"#"):
+                continue
+            clean.append(
+                raw.decode("utf-8", "ignore") if isinstance(raw, bytes) else raw
+            )
+
+        return StringIO("".join(clean))
+
+    def _build_skiprows(self):
+        return lambda idx: idx != self.header_row and idx < self.data_start_row

hydroserverpy/api/models/etl/transformers/json_transformer.py

@@ -0,0 +1,48 @@
+import logging
+import pandas as pd
+from typing import Optional, Any, List
+from .base import Transformer
+import json
+import jmespath
+from ..etl_configuration import TransformerConfig, SourceTargetMapping
+
+
+class JSONTransformer(Transformer):
+    def __init__(self, transformer_config: TransformerConfig):
+        super().__init__(transformer_config)
+        self.jmespath = transformer_config.jmespath
+
+    def transform(self, data_file, mappings: List[SourceTargetMapping]):
+        """
+        Transforms a JSON file-like object into the standard Pandas dataframe format.
+        Since JMESPath can natively rename column names, the assumption is the timestamp column
+        is always named 'timestamp' for JSON data or converted to 'timestamp' in the JMESPath query.
+
+        Parameters:
+            data_file: File-like object containing JSON data.
+
+        Returns:
+            pd.DataFrame: pandas DataFrames in the format pd.Timestamp, datastream_id_1, datastream_id_2, ...
+        """
+        if data_file is None:
+            raise TypeError(
+                "JSONTransformer received None; expected file-like, bytes, or str"
+            )
+
+        json_data = json.load(data_file)
+        data_points = self.extract_data_points(json_data)
+        if not data_points:
+            logging.warning("No data points found in the JSON data.")
+            return None
+
+        df = pd.DataFrame(data_points)
+
+        return self.standardize_dataframe(df, mappings)
+
+    def extract_data_points(self, json_data: Any) -> Optional[List[dict]]:
+        """Extracts data points from the JSON data using the data_path."""
+        data_points = jmespath.search(self.jmespath, json_data)
+
+        if isinstance(data_points, dict):
+            data_points = [data_points]
+        return data_points

hydroserverpy/api/models/etl/types.py

@@ -0,0 +1,7 @@
+from typing import TypedDict
+import pandas as pd
+
+
+class TimeRange(TypedDict):
+    start_time: pd.Timestamp
+    end_time: pd.Timestamp

hydroserverpy/api/models/iam/account.py

@@ -0,0 +1,12 @@
+from typing import Optional
+from pydantic import BaseModel, Field, EmailStr
+
+
+class Account(BaseModel):
+    name: str = Field(..., max_length=255)
+    email: EmailStr
+    organization_name: Optional[str] = None
+    phone: Optional[str] = Field(None, max_length=15)
+    address: Optional[str] = Field(None, max_length=255)
+    link: Optional[str] = Field(None, max_length=2000)
+    user_type: str = Field(..., max_length=255, alias="type")

hydroserverpy/api/models/iam/apikey.py

@@ -0,0 +1,96 @@
+from typing import Optional, Union, ClassVar, TYPE_CHECKING
+from uuid import UUID
+from datetime import datetime
+from hydroserverpy.api.models.iam.role import Role
+from hydroserverpy.api.utils import normalize_uuid
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class APIKey(HydroServerBaseModel):
+    name: str
+    role_id: Union[UUID, str]
+    workspace_id: Union[UUID, str]
+    description: Optional[str] = None
+    is_active: bool
+    expires_at: Optional[datetime] = None
+
+    _editable_fields: ClassVar[set[str]] = {"name", "description", "role_id", "is_active", "expires_at"}
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=None, **data)
+
+        self._workspace = None
+        self._role = None
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this API key belongs to."""
+
+        if self._workspace is None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def role(self) -> "Role":
+        """The role this API key is assigned."""
+
+        if self._role is None:
+            self._role = self.client.roles.get(uid=self.role_id)
+
+        return self._role
+
+    @role.setter
+    def role(self, role: Union["Role", UUID, str] = ...):
+        if not role:
+            raise ValueError("Role of API key cannot be None.")
+        if normalize_uuid(role) != str(self.role_id):
+            self.role_id = normalize_uuid(role)
+            self._role = None
+
+    def save(self):
+        """Saves changes to this resource to HydroServer."""
+
+        if not self.uid:
+            raise AttributeError("Data cannot be saved: UID is not set.")
+
+        if self.unsaved_changes:
+            saved_resource = self.client.workspaces.update_api_key(
+                uid=self.workspace_id, api_key_id=self.uid, **self.unsaved_changes
+            )
+            self._server_data = saved_resource.dict(by_alias=False).copy()
+            self.__dict__.update(saved_resource.__dict__)
+
+    def refresh(self):
+        """Refreshes this resource from HydroServer."""
+
+        if self.uid is None:
+            raise ValueError("Cannot refresh data without a valid ID.")
+
+        refreshed_resource = self.client.workspaces.get_api_key(uid=self.workspace_id, api_key_id=self.uid)
+        self._server_data = refreshed_resource.dict(by_alias=False).copy()
+        self.__dict__.update(refreshed_resource.__dict__)
+
+    def delete(self):
+        """Deletes this resource from HydroServer."""
+
+        if self.uid is None:
+            raise AttributeError("Cannot delete data without a valid ID.")
+
+        self.client.workspaces.delete_api_key(
+            uid=self.workspace_id, api_key_id=self.uid
+        )
+        self.uid = None
+
+    def regenerate(self):
+        """Regenerates this API key. WARNING: Previous key will be invalidated."""
+
+        _, key = self.client.workspaces.regenerate_api_key(
+            uid=self.workspace_id, api_key_id=self.uid
+        )
+
+        return key

hydroserverpy/api/models/iam/collaborator.py

@@ -0,0 +1,70 @@
+from typing import Union, ClassVar, TYPE_CHECKING
+from uuid import UUID
+from pydantic import Field, AliasPath
+from hydroserverpy.api.models.iam.role import Role
+from hydroserverpy.api.utils import normalize_uuid
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models.iam.workspace import Workspace
+    from hydroserverpy.api.models.iam.account import Account
+
+
+class Collaborator(HydroServerBaseModel):
+    user: "Account"
+    role_id: Union[UUID, str] = Field(..., validation_alias=AliasPath("role", "id"))
+    workspace_id: Union[UUID, str]
+
+    _editable_fields: ClassVar[set[str]] = {"role_id"}
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=None, **data)
+
+        self._workspace = None
+        self._role = Role(client=client, **data.get("role"))
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this collaborator belongs to."""
+
+        if self._workspace is None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def role(self) -> "Role":
+        """The role this collaborator is assigned."""
+
+        if self._role is None:
+            self._role = self.client.roles.get(uid=self.role_id)
+
+        return self._role
+
+    @role.setter
+    def role(self, role: Union["Role", UUID, str] = ...):
+        if not role:
+            raise ValueError("Role of collaborator cannot be None.")
+        if normalize_uuid(role) != str(self.role_id):
+            self.role_id = normalize_uuid(role)
+            self._role = None
+
+    def save(self):
+        """Saves changes to this resource to HydroServer."""
+
+        if self.unsaved_changes:
+            self.client.workspaces.edit_collaborator_role(
+                uid=str(self.workspace_id), email=self.user.email, role=self.role
+            )
+            self._role = None
+            self._server_data["role_id"] = self.role_id
+            self.__dict__.update({"role_id": self.role_id})
+
+    def delete(self):
+        """Deletes this resource from HydroServer."""
+
+        self.client.workspaces.remove_collaborator(
+            uid=str(self.workspace_id), email=self.user.email
+        )
+        self.uid = None

hydroserverpy/api/models/iam/role.py

@@ -0,0 +1,38 @@
+from typing import Optional, Union, TYPE_CHECKING
+from uuid import UUID
+from pydantic import Field
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class Role(HydroServerBaseModel):
+    name: str = Field(..., max_length=255)
+    description: str
+    workspace_id: Optional[Union[UUID, str]] = None
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.workspaces, **data)
+
+        self._workspace = None
+
+    @classmethod
+    def get_route(cls):
+        return "roles"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this role belongs to."""
+
+        if self._workspace is None and self.workspace_id is not None:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    def save(self):
+        raise NotImplementedError("Editing roles not enabled.")
+
+    def delete(self):
+        raise NotImplementedError("Deleting roles not enabled.")