hydroserverpy 1.5.1__py3-none-any.whl

hydroserverpy/api/models/etl/etl_configuration.py

@@ -0,0 +1,224 @@
+from typing import Annotated, Dict, List, Literal, Optional, Union
+from pydantic import BaseModel, Field, field_validator
+from enum import Enum
+
+WorkflowType = Literal["ETL", "Aggregation", "Virtual", "SDL"]
+CSVDelimiterType = Literal[",", "|", "\t", ";", " "]
+ExtractorType = Literal["HTTP", "local"]
+TransformerType = Literal["JSON", "CSV"]
+LoaderType = Literal["HydroServer"]
+IdentifierType = Literal["name", "index"]
+RunTimeValue = Literal["jobExecutionTime", "latestObservationTimestamp"]
+
+
+class FixedOffsetTimezone(str, Enum):
+    UTC_MINUS_1200 = "-1200"
+    UTC_MINUS_1100 = "-1100"
+    UTC_MINUS_1000 = "-1000"
+    UTC_MINUS_0900 = "-0900"
+    UTC_MINUS_0800 = "-0800"
+    UTC_MINUS_0700 = "-0700"
+    UTC_MINUS_0600 = "-0600"
+    UTC_MINUS_0500 = "-0500"
+    UTC_MINUS_0430 = "-0430"
+    UTC_MINUS_0400 = "-0400"
+    UTC_MINUS_0330 = "-0330"
+    UTC_MINUS_0300 = "-0300"
+    UTC_MINUS_0200 = "-0200"
+    UTC_MINUS_0100 = "-0100"
+    UTC_PLUS_0000 = "+0000"
+    UTC_PLUS_0100 = "+0100"
+    UTC_PLUS_0200 = "+0200"
+    UTC_PLUS_0300 = "+0300"
+    UTC_PLUS_0330 = "+0330"
+    UTC_PLUS_0400 = "+0400"
+    UTC_PLUS_0430 = "+0430"
+    UTC_PLUS_0500 = "+0500"
+    UTC_PLUS_0530 = "+0530"
+    UTC_PLUS_0545 = "+0545"
+    UTC_PLUS_0600 = "+0600"
+    UTC_PLUS_0630 = "+0630"
+    UTC_PLUS_0700 = "+0700"
+    UTC_PLUS_0800 = "+0800"
+    UTC_PLUS_0845 = "+0845"
+    UTC_PLUS_0900 = "+0900"
+    UTC_PLUS_0930 = "+0930"
+    UTC_PLUS_1000 = "+1000"
+    UTC_PLUS_1030 = "+1030"
+    UTC_PLUS_1100 = "+1100"
+    UTC_PLUS_1130 = "+1130"
+    UTC_PLUS_1200 = "+1200"
+    UTC_PLUS_1245 = "+1245"
+    UTC_PLUS_1300 = "+1300"
+    UTC_PLUS_1400 = "+1400"
+
+
+class TimestampFormat(str, Enum):
+    ISO8601 = "ISO8601"
+    naive = "naive"
+    custom = "custom"
+
+
+class TimezoneMode(str, Enum):
+    utc = "utc"  # always UTC
+    daylightSavings = "daylightSavings"  # IANA / DST-aware
+    fixedOffset = "fixedOffset"  # constant offset
+    embeddedOffset = "embeddedOffset"  # offset in ISO string
+
+
+class Timestamp(BaseModel):
+    key: Optional[str] = None
+    format: TimestampFormat
+    custom_format: Optional[str] = Field(None, alias="customFormat")
+    timezone_mode: TimezoneMode = Field(..., alias="timezoneMode")
+    timezone: Optional[Union[FixedOffsetTimezone, str]] = Field(None, alias="timezone")
+
+    class Config:
+        populate_by_name = True
+
+    @field_validator("timezone")
+    def check_timezone(cls, timezone_value, info):
+        mode = info.data.get("timezone_mode")
+        if mode == TimezoneMode.fixedOffset and timezone_value is None:
+            raise ValueError("`timezone` must be set when timezoneMode is fixedOffset")
+        return timezone_value
+
+
+class PerPayloadPlaceholder(BaseModel):
+    name: str
+    type: Literal["perPayload"]
+
+
+class RunTimePlaceholder(BaseModel):
+    name: str
+    type: Literal["runTime"]
+    run_time_value: RunTimeValue = Field(..., alias="runTimeValue")
+    timestamp: Timestamp
+
+    class Config:
+        populate_by_name = True
+
+
+PlaceholderVariable = Annotated[
+    Union[PerPayloadPlaceholder, RunTimePlaceholder],
+    Field(discriminator="type"),
+]
+
+
+class BaseExtractor(BaseModel):
+    type: ExtractorType
+    source_uri: str = Field(..., alias="sourceUri")
+    placeholder_variables: Optional[List[PlaceholderVariable]] = Field(
+        default_factory=list,
+        alias="placeholderVariables",
+    )
+
+    class Config:
+        populate_by_name = True
+
+
+class HTTPExtractor(BaseExtractor):
+    type: Literal["HTTP"]
+
+
+class LocalFileExtractor(BaseExtractor):
+    type: Literal["local"]
+
+
+ExtractorConfig = Annotated[
+    Union[HTTPExtractor, LocalFileExtractor], Field(discriminator="type")
+]
+
+
+class BaseTransformer(BaseModel):
+    type: TransformerType
+    timestamp: Timestamp
+
+
+class JSONTransformer(BaseTransformer):
+    type: Literal["JSON"]
+    jmespath: str = Field(..., alias="JMESPath")
+
+    class Config:
+        populate_by_name = True
+
+
+class CSVTransformer(BaseTransformer):
+    type: Literal["CSV"]
+    header_row: Optional[int] = Field(..., alias="headerRow")
+    data_start_row: int = Field(..., alias="dataStartRow")
+    delimiter: CSVDelimiterType
+    identifier_type: IdentifierType = Field(..., alias="identifierType")
+
+    class Config:
+        populate_by_name = True
+
+
+TransformerConfig = Union[JSONTransformer, CSVTransformer]
+
+
+class BaseLoaderConfig(BaseModel):
+    type: LoaderType
+
+
+class HydroServerLoaderConfig(BaseLoaderConfig):
+    type: Literal["HydroServer"]
+
+
+LoaderConfig = HydroServerLoaderConfig
+
+
+class ExpressionDataTransformation(BaseModel):
+    type: Literal["expression"]
+    expression: str
+
+    class Config:
+        populate_by_name = True
+
+
+class LookupTableDataTransformation(BaseModel):
+    type: Literal["lookup"]
+    lookup_table_id: str = Field(..., alias="lookupTableId")
+
+    class Config:
+        populate_by_name = True
+
+
+DataTransformation = Union[ExpressionDataTransformation, LookupTableDataTransformation]
+
+
+class MappingPath(BaseModel):
+    target_identifier: Union[str, int] = Field(..., alias="targetIdentifier")
+    data_transformations: List[DataTransformation] = Field(
+        default_factory=list, alias="dataTransformations"
+    )
+
+    class Config:
+        populate_by_name = True
+
+
+class SourceTargetMapping(BaseModel):
+    source_identifier: Union[str, int] = Field(..., alias="sourceIdentifier")
+    paths: List[MappingPath] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+
+
+class Payload(BaseModel):
+    name: str = ""
+    mappings: List[SourceTargetMapping] = Field(default_factory=list)
+    extractor_variables: Dict[str, str] = Field(
+        default_factory=dict, alias="extractorVariables"
+    )
+
+    class Config:
+        populate_by_name = True
+
+
+class EtlConfiguration(BaseModel):
+    type: WorkflowType
+    extractor: ExtractorConfig
+    transformer: TransformerConfig
+    loader: LoaderConfig
+    payloads: List[Payload]

hydroserverpy/api/models/etl/extractors/__init__.py

@@ -0,0 +1,6 @@
+from .base import Extractor
+from .ftp_extractor import FTPExtractor
+from .http_extractor import HTTPExtractor
+from .local_file_extractor import LocalFileExtractor
+
+__all__ = ["Extractor", "HTTPExtractor", "LocalFileExtractor", "FTPExtractor"]

hydroserverpy/api/models/etl/extractors/base.py

@@ -0,0 +1,52 @@
+from abc import abstractmethod
+import logging
+import pandas as pd
+from datetime import datetime
+from ..etl_configuration import ExtractorConfig, Payload
+from ..timestamp_parser import TimestampParser
+
+
+class Extractor:
+    def __init__(self, extractor_config: ExtractorConfig):
+        self.cfg = extractor_config
+
+    def resolve_placeholder_variables(self, payload: Payload, loader):
+        logging.info(f"Creating runtime variables...")
+        filled = {}
+        for placeholder in self.cfg.placeholder_variables:
+            name = placeholder.name
+
+            if placeholder.type == "runTime":
+                logging.info(f"Resolving runtime var: {name}")
+                if placeholder.run_time_value == "latestObservationTimestamp":
+                    value = loader.earliest_begin_date(payload)
+                elif placeholder.run_time_value == "jobExecutionTime":
+                    value = pd.Timestamp.now(tz="UTC")
+            elif placeholder.type == "perPayload":
+                logging.info(f"Resolving payload var: {name}")
+                if name not in payload.extractor_variables:
+                    raise KeyError(f"Missing per-payload variable '{name}'")
+                value = payload.extractor_variables[name]
+            else:
+                continue
+
+            if isinstance(value, (datetime, pd.Timestamp)):
+                parser = TimestampParser(placeholder.timestamp)
+                value = parser.utc_to_string(value)
+
+            filled[name] = value
+        if not filled:
+            return self.cfg.source_uri
+        return self.format_uri(filled)
+
+    def format_uri(self, placeholder_variables):
+        try:
+            uri = self.cfg.source_uri.format(**placeholder_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing placeholder variable: {missing_key}")
+        return uri
+
+    @abstractmethod
+    def extract(self):
+        pass

hydroserverpy/api/models/etl/extractors/ftp_extractor.py

@@ -0,0 +1,50 @@
+import logging
+from ftplib import FTP
+from io import BytesIO
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class FTPExtractor(Extractor):
+    def __init__(
+        self,
+        host: str,
+        filepath: str,
+        username: str = None,
+        password: str = None,
+        port: int = 21,
+    ):
+        self.host = host
+        self.port = int(port)
+        self.username = username
+        self.password = password
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Downloads the file from the FTP server and returns a file-like object.
+        """
+        ftp = FTP()
+        try:
+            ftp.connect(self.host, self.port)
+            ftp.login(user=self.username, passwd=self.password)
+            logging.info(f"Connected to FTP server: {self.host}:{self.port}")
+
+            data = BytesIO()
+            ftp.retrbinary(f"RETR {self.filepath}", data.write)
+            logging.info(
+                f"Successfully downloaded file '{self.filepath}' from FTP server."
+            )
+            data.seek(0)
+            return data
+        except Exception as e:
+            logging.error(f"Error retrieving file from FTP server: {e}")
+            return None
+        finally:
+            if ftp:
+                ftp.quit()

hydroserverpy/api/models/etl/extractors/http_extractor.py

@@ -0,0 +1,28 @@
+import logging
+import requests
+from io import BytesIO
+
+from ..etl_configuration import Payload
+from .base import Extractor, ExtractorConfig
+
+
+class HTTPExtractor(Extractor):
+    def __init__(self, settings: ExtractorConfig):
+        super().__init__(settings)
+
+    def extract(self, payload: Payload, loader=None):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+        url = self.resolve_placeholder_variables(payload, loader)
+        logging.info(f"Requesting data from → {url}")
+
+        response = requests.get(url)
+        response.raise_for_status()
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data

hydroserverpy/api/models/etl/extractors/local_file_extractor.py

@@ -0,0 +1,20 @@
+import logging
+from .base import Extractor
+from ..etl_configuration import ExtractorConfig
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, extractor_config: ExtractorConfig):
+        super().__init__(extractor_config)
+
+    def extract(self, *args, **kwargs):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.cfg.source_uri, "r")
+            logging.info(f"Successfully opened file '{self.cfg.source_uri}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.cfg.source_uri}': {e}")
+            return None

hydroserverpy/api/models/etl/factories.py

@@ -0,0 +1,23 @@
+from .extractors import HTTPExtractor, LocalFileExtractor
+from .transformers import JSONTransformer, CSVTransformer
+from .loaders import HydroServerLoader
+from .etl_configuration import ExtractorConfig, TransformerConfig, LoaderConfig
+
+EXTRACTORS = {"HTTP": HTTPExtractor, "local": LocalFileExtractor}
+TRANSFORMERS = {"JSON": JSONTransformer, "CSV": CSVTransformer}
+LOADERS = {"HydroServer": HydroServerLoader}
+
+
+def extractor_factory(settings: ExtractorConfig):
+    cls = EXTRACTORS[settings.type]
+    return cls(settings)
+
+
+def transformer_factory(settings: TransformerConfig):
+    cls = TRANSFORMERS[settings.type]
+    return cls(settings)
+
+
+def loader_factory(settings: LoaderConfig, auth_context, data_source_id: str):
+    cls = LOADERS[settings.type]
+    return cls(auth_context, data_source_id)

hydroserverpy/api/models/etl/loaders/__init__.py

@@ -0,0 +1,4 @@
+from .base import Loader
+from .hydroserver_loader import HydroServerLoader
+
+__all__ = ["Loader", "HydroServerLoader"]

hydroserverpy/api/models/etl/loaders/base.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class Loader(ABC):
+    @abstractmethod
+    def load(self, *args, **kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def earliest_begin_date(self, payload_mappings) -> str:
+        pass

hydroserverpy/api/models/etl/loaders/hydroserver_loader.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from .base import Loader
+import logging
+import pandas as pd
+from ..etl_configuration import Payload, SourceTargetMapping
+
+if TYPE_CHECKING:
+    from hydroserverpy.api.client import HydroServer
+
+
+class HydroServerLoader(Loader):
+    """
+    A class that extends the HydroServer client with ETL-specific functionalities.
+    """
+
+    def __init__(self, client: HydroServer, data_source_id):
+        self.client = client
+        self._begin_cache: dict[str, pd.Timestamp] = {}
+        self.data_source_id = data_source_id
+
+    def load(self, data: pd.DataFrame, payload: Payload) -> None:
+        """
+        Load observations from a DataFrame to the HydroServer.
+        :param data: A Pandas DataFrame where each column corresponds to a datastream.
+        """
+        begin_date = self.earliest_begin_date(payload)
+        new_data = data[data["timestamp"] > begin_date]
+        for col in new_data.columns.difference(["timestamp"]):
+            df = (
+                new_data[["timestamp", col]]
+                .rename(columns={col: "value"})
+                .dropna(subset=["value"])
+            )
+            if df.empty:
+                logging.warning(f"No new data for {col}, skipping.")
+                continue
+
+            df = df.rename(columns={"timestamp": "phenomenon_time", "value": "result"})
+
+            # Chunked upload
+            CHUNK_SIZE = 5000
+            total = len(df)
+            for start in range(0, total, CHUNK_SIZE):
+                end = min(start + CHUNK_SIZE, total)
+                chunk = df.iloc[start:end]
+                logging.info(
+                    "Uploading %s rows (%s-%s) to datastream %s",
+                    len(chunk),
+                    start,
+                    end - 1,
+                    col,
+                )
+                try:
+                    self.client.datastreams.load_observations(
+                        uid=str(col), observations=chunk
+                    )
+                except Exception as e:
+                    status = getattr(e, "status_code", None) or getattr(
+                        getattr(e, "response", None), "status_code", None
+                    )
+                    if status == 409 or "409" in str(e) or "Conflict" in str(e):
+                        logging.info(
+                            "409 Conflict for datastream %s on rows %s-%s; skipping remainder for this stream.",
+                            col,
+                            start,
+                            end - 1,
+                        )
+                        break
+                    raise
+
+    def _fetch_earliest_begin(
+        self, mappings: list[SourceTargetMapping]
+    ) -> pd.Timestamp:
+        logging.info("Querying HydroServer for earliest begin date for payload...")
+        timestamps = []
+        datastreams = self.client.datastreams.list(
+            data_source=self.data_source_id
+        ).items
+        ds_by_uid = {str(ds.uid): ds for ds in datastreams}
+        for m in mappings:
+            for p in m.paths:
+                datastream = ds_by_uid[str(p.target_identifier)]
+                raw = datastream.phenomenon_end_time or "1970-01-01"
+                ts = pd.to_datetime(raw, utc=True)
+                timestamps.append(ts)
+        logging.info(f"Found earliest begin date: {min(timestamps)}")
+        return min(timestamps)
+
+    def earliest_begin_date(self, payload: Payload) -> pd.Timestamp:
+        """
+        Return earliest begin date for a payload, or compute+cache it on first call.
+        """
+        key = payload.name
+        if key not in self._begin_cache:
+            self._begin_cache[key] = self._fetch_earliest_begin(payload.mappings)
+        return self._begin_cache[key]

hydroserverpy/api/models/etl/orchestration_configuration.py

@@ -0,0 +1,35 @@
+from pydantic import AliasPath
+from typing import Optional, Literal
+from datetime import datetime
+from pydantic import BaseModel, Field
+
+
+class OrchestrationConfigurationFields(BaseModel):
+    interval: Optional[int] = Field(
+        None, gt=0, validation_alias=AliasPath("schedule", "interval")
+    )
+    interval_units: Optional[Literal["minutes", "hours", "days"]] = Field(
+        None, validation_alias=AliasPath("schedule", "intervalUnits")
+    )
+    crontab: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("schedule", "crontab")
+    )
+    start_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "startTime")
+    )
+    end_time: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("schedule", "endTime")
+    )
+    last_run_successful: Optional[bool] = Field(
+        None, validation_alias=AliasPath("status", "lastRunSuccessful")
+    )
+    last_run_message: Optional[str] = Field(
+        None, max_length=255, validation_alias=AliasPath("status", "lastRunMessage")
+    )
+    last_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "lastRun")
+    )
+    next_run: Optional[datetime] = Field(
+        None, validation_alias=AliasPath("status", "nextRun")
+    )
+    paused: bool = Field(False, validation_alias=AliasPath("status", "paused"))

hydroserverpy/api/models/etl/orchestration_system.py

@@ -0,0 +1,63 @@
+import uuid
+from typing import Optional, ClassVar, List, TYPE_CHECKING
+from pydantic import BaseModel, Field
+from ..base import HydroServerBaseModel
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace, DataSource, DataArchive
+
+
+class OrchestrationSystemFields(BaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+
+
+class OrchestrationSystem(HydroServerBaseModel):
+    name: str = Field(..., max_length=255)
+    orchestration_system_type: str = Field(..., max_length=255, alias="type")
+    workspace_id: Optional[uuid.UUID] = None
+
+    _editable_fields: ClassVar[set[str]] = {"name", "orchestration_system_type"}
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.orchestrationsystems, **data)
+
+        self._workspace = None
+        self._datasources = None
+        self._dataarchives = None
+
+    @classmethod
+    def get_route(cls):
+        return "orchestration-systems"
+
+    @property
+    def workspace(self) -> "Workspace":
+        """The workspace this orchestration system belongs to."""
+
+        if self._workspace is None and self.workspace_id:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
+
+        return self._workspace
+
+    @property
+    def datasources(self) -> List["DataSource"]:
+        """The data sources associated with this workspace."""
+
+        if self._datasources is None:
+            self._datasources = self.client.datasources.list(
+                orchestration_system=self.uid, fetch_all=True
+            ).items
+
+        return self._datasources
+
+    @property
+    def dataarchives(self) -> List["DataArchive"]:
+        """The data archives associated with this workspace."""
+
+        if self._dataarchives is None:
+            self._dataarchives = self.client.dataarchives.list(
+                orchestration_system=self.uid, fetch_all=True
+            ).items
+
+        return self._dataarchives

hydroserverpy/api/models/etl/schedule.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from typing import Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class Schedule(BaseModel):
+    interval: int = Field(..., gt=0)
+    interval_units: Optional[Literal["minutes", "hours", "days"]] = Field(
+        None, alias="intervalUnits"
+    )
+    crontab: Optional[str]
+    start_time: Optional[datetime] = Field(None, alias="startTime")
+    end_time: Optional[datetime] = Field(None, alias="endTime")
+
+    class Config:
+        populate_by_name = True

hydroserverpy/api/models/etl/status.py

@@ -0,0 +1,14 @@
+from datetime import datetime
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class Status(BaseModel):
+    paused: bool = Field(False)
+    last_run_successful: Optional[bool] = Field(None, alias="lastRunSuccessful")
+    last_run_message: Optional[str] = Field(None, alias="lastRunMessage")
+    last_run: Optional[datetime] = Field(None, alias="lastRun")
+    next_run: Optional[datetime] = Field(None, alias="nextRun")
+
+    class Config:
+        populate_by_name = True