hydroserverpy 1.3.0b3__tar.gz → 1.4.0b3__tar.gz

{hydroserverpy-1.3.0b3/src/hydroserverpy.egg-info → hydroserverpy-1.4.0b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.3.0b3
+Version: 1.4.0b3
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.3.0b3 → hydroserverpy-1.4.0b3}/README.md

@@ -40,4 +40,5 @@ hs_api = HydroServer(
 
 ## Funding and Acknowledgements
 
-Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama (NA22NWS4320003).
+Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama (NA22NWS4320003). Utah State University is a founding member of CIROH and receives funding under subaward from the University of Alabama. Additional funding and support have been provided by the State of Utah Division of Water Rights, the World Meorological Organization, and the Utah Water Research laboratory at Utah State University.
+

{hydroserverpy-1.3.0b3 → hydroserverpy-1.4.0b3}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = 1.3.0b3
+version = 1.4.0b3
 
 [options]
 package_dir = 

{hydroserverpy-1.3.0b3 → hydroserverpy-1.4.0b3}/src/hydroserverpy/__init__.py

@@ -1,9 +1,7 @@
 from .api.client import HydroServer
-from .etl.hydroserver_etl import HydroServerETL
 from .quality import HydroServerQualityControl
 
 __all__ = [
     "HydroServer",
     "HydroServerQualityControl",
-    "HydroServerETL",
 ]

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/__init__.py

@@ -0,0 +1,26 @@
+from .extractors import Extractor, HTTPExtractor, LocalFileExtractor, FTPExtractor
+from .transformers import JSONTransformer, CSVTransformer, Transformer
+from .loaders import HydroServerLoader, Loader
+
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .orchestration_system import OrchestrationSystem
+from .data_source import DataSource
+
+__all__ = [
+    "CSVTransformer",
+    "JSONTransformer",
+    "LocalFileExtractor",
+    "FTPExtractor",
+    "HTTPExtractor",
+    "Extractor",
+    "Transformer",
+    "Loader",
+    "HydroServerLoader",
+    "EtlConfiguration",
+    "Schedule",
+    "Status",
+    "OrchestrationSystem",
+    "DataSource",
+]

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/data_source.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+from datetime import datetime, timedelta, timezone
+from functools import cached_property
+import logging
+import uuid
+from typing import ClassVar, TYPE_CHECKING, List, Optional, Union
+import croniter
+import pandas as pd
+from pydantic import Field
+
+from ..base import HydroServerBaseModel
+from ..sta.datastream import Datastream
+from .orchestration_system import OrchestrationSystem
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .factories import extractor_factory, transformer_factory, loader_factory
+from .loaders import HydroServerLoader
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class DataSource(HydroServerBaseModel):
+    name: str = Field(..., max_length=255)
+    settings: EtlConfiguration
+    orchestration_system_id: uuid.UUID
+    schedule: Schedule
+    status: Status
+    workspace_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name",
+        "settings",
+        "status",
+        "schedule",
+        "interval",
+        "interval_units",
+        "crontab",
+        "start_time",
+        "end_time",
+        "last_run_successful",
+        "last_run_message",
+        "last_run",
+        "next_run",
+        "paused",
+    }
+
+    def __init__(self, client: HydroServer, **data):
+        super().__init__(client=client, service=client.datasources, **data)
+
+    @classmethod
+    def get_route(cls):
+        return "data-sources"
+
+    @cached_property
+    def workspace(self) -> Workspace:
+        return self.client.workspaces.get(uid=self.workspace_id)
+
+    @cached_property
+    def orchestration_system(self) -> OrchestrationSystem:
+        return self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+
+    @cached_property
+    def datastreams(self) -> List[Datastream]:
+        return self.client.datastreams.list(data_source=self.uid, fetch_all=True).items
+
+    # TODO: Add functions like add_payload, add_mapping, etc. and don't allow the user to manually
+    # link or unlink datastreams - handle that automatically.
+    def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Add a datastream to this data source."""
+
+        self.client.datasources.add_datastream(uid=self.uid, datastream=datastream)
+
+    def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
+        """Remove a datastream from this data source."""
+
+        self.client.datasources.remove_datastream(uid=self.uid, datastream=datastream)
+
+    def _next_run(self) -> Optional[str]:
+        now = datetime.now(timezone.utc)
+        if cron := self.schedule.crontab:
+            return croniter.croniter(cron, now).get_next(datetime).isoformat()
+        if iv := self.schedule.interval:
+            unit = self.schedule.interval_units or "minutes"
+            return (now + timedelta(**{unit: iv})).isoformat()
+        return None
+
+    def _update_status(self, loader: HydroServerLoader, success: bool, msg: str):
+        short_msg = msg if len(msg) <= 255 else msg[:252] + "…"
+        loader.client.datasources.update(
+            uid=self.uid,
+            last_run=datetime.now(timezone.utc).isoformat(),
+            last_run_successful=success,
+            last_run_message=short_msg,
+            next_run=self._next_run(),
+        )
+
+    def is_empty(self, data):
+        if data is None:
+            return True
+        if isinstance(data, pd.DataFrame) and data.empty:
+            return True
+        return False
+
+    def load_data(self, payload_name: str = None):
+        """Load data for this data source."""
+        if self.status.paused is True:
+            return
+
+        if payload_name:
+            self.load_data_for_payload(payload_name)
+        else:
+            for p in self.settings.payloads:
+                self.load_data_for_payload(p.name)
+
+    def load_data_for_payload(self, payload_name: str):
+        payload = next(p for p in self.settings.payloads if p.name == payload_name)
+
+        extractor_cls = extractor_factory(self.settings.extractor)
+        transformer_cls = transformer_factory(self.settings.transformer)
+        loader_cls = loader_factory(self.settings.loader, self.client, self.uid)
+
+        try:
+            logging.info("Starting extract")
+            data = extractor_cls.extract(payload, loader_cls)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the extractor"
+                )
+                return
+
+            logging.info("Starting transform")
+            data = transformer_cls.transform(data, payload.mappings)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the transformer"
+                )
+                return
+
+            logging.info("Starting load")
+            loader_cls.load(data, payload)
+            self._update_status(loader_cls, True, "OK")
+        except Exception as e:
+            self._update_status(loader_cls, False, str(e))

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/etl_configuration.py

@@ -0,0 +1,224 @@
+from typing import Annotated, Dict, List, Literal, Optional, Union
+from pydantic import BaseModel, Field, field_validator
+from enum import Enum
+
+WorkflowType = Literal["ETL", "Aggregation", "Virtual", "SDL"]
+CSVDelimiterType = Literal[",", "|", "\t", ";", " "]
+ExtractorType = Literal["HTTP", "local"]
+TransformerType = Literal["JSON", "CSV"]
+LoaderType = Literal["HydroServer"]
+IdentifierType = Literal["name", "index"]
+RunTimeValue = Literal["jobExecutionTime", "latestObservationTimestamp"]
+
+
+class FixedOffsetTimezone(str, Enum):
+    UTC_MINUS_1200 = "-1200"
+    UTC_MINUS_1100 = "-1100"
+    UTC_MINUS_1000 = "-1000"
+    UTC_MINUS_0900 = "-0900"
+    UTC_MINUS_0800 = "-0800"
+    UTC_MINUS_0700 = "-0700"
+    UTC_MINUS_0600 = "-0600"
+    UTC_MINUS_0500 = "-0500"
+    UTC_MINUS_0430 = "-0430"
+    UTC_MINUS_0400 = "-0400"
+    UTC_MINUS_0330 = "-0330"
+    UTC_MINUS_0300 = "-0300"
+    UTC_MINUS_0200 = "-0200"
+    UTC_MINUS_0100 = "-0100"
+    UTC_PLUS_0000 = "+0000"
+    UTC_PLUS_0100 = "+0100"
+    UTC_PLUS_0200 = "+0200"
+    UTC_PLUS_0300 = "+0300"
+    UTC_PLUS_0330 = "+0330"
+    UTC_PLUS_0400 = "+0400"
+    UTC_PLUS_0430 = "+0430"
+    UTC_PLUS_0500 = "+0500"
+    UTC_PLUS_0530 = "+0530"
+    UTC_PLUS_0545 = "+0545"
+    UTC_PLUS_0600 = "+0600"
+    UTC_PLUS_0630 = "+0630"
+    UTC_PLUS_0700 = "+0700"
+    UTC_PLUS_0800 = "+0800"
+    UTC_PLUS_0845 = "+0845"
+    UTC_PLUS_0900 = "+0900"
+    UTC_PLUS_0930 = "+0930"
+    UTC_PLUS_1000 = "+1000"
+    UTC_PLUS_1030 = "+1030"
+    UTC_PLUS_1100 = "+1100"
+    UTC_PLUS_1130 = "+1130"
+    UTC_PLUS_1200 = "+1200"
+    UTC_PLUS_1245 = "+1245"
+    UTC_PLUS_1300 = "+1300"
+    UTC_PLUS_1400 = "+1400"
+
+
+class TimestampFormat(str, Enum):
+    ISO8601 = "ISO8601"
+    naive = "naive"
+    custom = "custom"
+
+
+class TimezoneMode(str, Enum):
+    utc = "utc"  # always UTC
+    daylightSavings = "daylightSavings"  # IANA / DST-aware
+    fixedOffset = "fixedOffset"  # constant offset
+    embeddedOffset = "embeddedOffset"  # offset in ISO string
+
+
+class Timestamp(BaseModel):
+    key: Optional[str] = None
+    format: TimestampFormat
+    custom_format: Optional[str] = Field(None, alias="customFormat")
+    timezone_mode: TimezoneMode = Field(..., alias="timezoneMode")
+    timezone: Optional[Union[FixedOffsetTimezone, str]] = Field(None, alias="timezone")
+
+    class Config:
+        allow_population_by_field_name = True
+
+    @field_validator("timezone")
+    def check_timezone(cls, timezone_value, info):
+        mode = info.data.get("timezone_mode")
+        if mode == TimezoneMode.fixedOffset and timezone_value is None:
+            raise ValueError("`timezone` must be set when timezoneMode is fixedOffset")
+        return timezone_value
+
+
+class PerPayloadPlaceholder(BaseModel):
+    name: str
+    type: Literal["perPayload"]
+
+
+class RunTimePlaceholder(BaseModel):
+    name: str
+    type: Literal["runTime"]
+    run_time_value: RunTimeValue = Field(..., alias="runTimeValue")
+    timestamp: Timestamp
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+PlaceholderVariable = Annotated[
+    Union[PerPayloadPlaceholder, RunTimePlaceholder],
+    Field(discriminator="type"),
+]
+
+
+class BaseExtractor(BaseModel):
+    type: ExtractorType
+    source_uri: str = Field(..., alias="sourceUri")
+    placeholder_variables: Optional[List[PlaceholderVariable]] = Field(
+        default_factory=list,
+        alias="placeholderVariables",
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class HTTPExtractor(BaseExtractor):
+    type: Literal["HTTP"]
+
+
+class LocalFileExtractor(BaseExtractor):
+    type: Literal["local"]
+
+
+ExtractorConfig = Annotated[
+    Union[HTTPExtractor, LocalFileExtractor], Field(discriminator="type")
+]
+
+
+class BaseTransformer(BaseModel):
+    type: TransformerType
+    timestamp: Timestamp
+
+
+class JSONTransformer(BaseTransformer):
+    type: Literal["JSON"]
+    jmespath: str = Field(..., alias="JMESPath")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class CSVTransformer(BaseTransformer):
+    type: Literal["CSV"]
+    header_row: Optional[int] = Field(..., alias="headerRow")
+    data_start_row: int = Field(..., alias="dataStartRow")
+    delimiter: CSVDelimiterType
+    identifier_type: IdentifierType = Field(..., alias="identifierType")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+TransformerConfig = Union[JSONTransformer, CSVTransformer]
+
+
+class BaseLoaderConfig(BaseModel):
+    type: LoaderType
+
+
+class HydroServerLoaderConfig(BaseLoaderConfig):
+    type: Literal["HydroServer"]
+
+
+LoaderConfig = HydroServerLoaderConfig
+
+
+class ExpressionDataTransformation(BaseModel):
+    type: Literal["expression"]
+    expression: str
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class LookupTableDataTransformation(BaseModel):
+    type: Literal["lookup"]
+    lookup_table_id: str = Field(..., alias="lookupTableId")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+DataTransformation = Union[ExpressionDataTransformation, LookupTableDataTransformation]
+
+
+class MappingPath(BaseModel):
+    target_identifier: Union[str, int] = Field(..., alias="targetIdentifier")
+    data_transformations: List[DataTransformation] = Field(
+        default_factory=list, alias="dataTransformations"
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class SourceTargetMapping(BaseModel):
+    source_identifier: Union[str, int] = Field(..., alias="sourceIdentifier")
+    paths: List[MappingPath] = Field(default_factory=list)
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Payload(BaseModel):
+    name: str = ""
+    mappings: List[SourceTargetMapping] = Field(default_factory=list)
+    extractor_variables: Dict[str, str] = Field(
+        default_factory=dict, alias="extractorVariables"
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class EtlConfiguration(BaseModel):
+    type: WorkflowType
+    extractor: ExtractorConfig
+    transformer: TransformerConfig
+    loader: LoaderConfig
+    payloads: List[Payload]

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/extractors/__init__.py

@@ -0,0 +1,6 @@
+from .base import Extractor
+from .ftp_extractor import FTPExtractor
+from .http_extractor import HTTPExtractor
+from .local_file_extractor import LocalFileExtractor
+
+__all__ = ["Extractor", "HTTPExtractor", "LocalFileExtractor", "FTPExtractor"]

{hydroserverpy-1.3.0b3/src/hydroserverpy → hydroserverpy-1.4.0b3/src/hydroserverpy/api/models}/etl/extractors/base.py

@@ -2,49 +2,46 @@ from abc import abstractmethod
 import logging
 import pandas as pd
 from datetime import datetime
-
-from hydroserverpy.etl.timestamp_parser import TimestampParser
+from ..etl_configuration import ExtractorConfig, Payload
+from ..timestamp_parser import TimestampParser
 
 
 class Extractor:
-    def __init__(self, settings: dict):
-        self.settings = settings
-        self.source_uri = settings["sourceUri"]
+    def __init__(self, extractor_config: ExtractorConfig):
+        self.cfg = extractor_config
 
-    def resolve_placeholder_variables(self, payload, loader):
+    def resolve_placeholder_variables(self, payload: Payload, loader):
         logging.info(f"Creating runtime variables...")
         filled = {}
-        for var in self.settings.get("placeholderVariables", []):
-            name = var["name"]
-            var_type = var.get("type", None)
+        for placeholder in self.cfg.placeholder_variables:
+            name = placeholder.name
 
-            if var_type == "runTime":
+            if placeholder.type == "runTime":
                 logging.info(f"Resolving runtime var: {name}")
-                if var.get("runTimeValue", None) == "latestObservationTimestamp":
+                if placeholder.run_time_value == "latestObservationTimestamp":
                     value = loader.earliest_begin_date(payload)
-                elif var.get("runTimeValue", None) == "jobExecutionTime":
+                elif placeholder.run_time_value == "jobExecutionTime":
                     value = pd.Timestamp.now(tz="UTC")
-            elif var_type == "perPayload":
+            elif placeholder.type == "perPayload":
                 logging.info(f"Resolving payload var: {name}")
-                payload_vars = payload.get("extractorVariables", {})
-                if name not in payload_vars:
+                if name not in payload.extractor_variables:
                     raise KeyError(f"Missing per-payload variable '{name}'")
-                value = payload_vars[name]
+                value = payload.extractor_variables[name]
             else:
                 continue
 
             if isinstance(value, (datetime, pd.Timestamp)):
-                parser = TimestampParser(var["timestamp"])
+                parser = TimestampParser(placeholder.timestamp)
                 value = parser.utc_to_string(value)
 
             filled[name] = value
         if not filled:
-            return self.source_uri
+            return self.cfg.source_uri
         return self.format_uri(filled)
 
     def format_uri(self, placeholder_variables):
         try:
-            uri = self.source_uri.format(**placeholder_variables)
+            uri = self.cfg.source_uri.format(**placeholder_variables)
         except KeyError as e:
             missing_key = e.args[0]
             raise KeyError(f"Missing placeholder variable: {missing_key}")

{hydroserverpy-1.3.0b3/src/hydroserverpy → hydroserverpy-1.4.0b3/src/hydroserverpy/api/models}/etl/extractors/http_extractor.py

@@ -1,14 +1,16 @@
 import logging
 import requests
 from io import BytesIO
-from .base import Extractor
+
+from ..etl_configuration import Payload
+from .base import Extractor, ExtractorConfig
 
 
 class HTTPExtractor(Extractor):
-    def __init__(self, settings: dict):
+    def __init__(self, settings: ExtractorConfig):
         super().__init__(settings)
 
-    def extract(self, payload, loader=None):
+    def extract(self, payload: Payload, loader=None):
         """
         Downloads the file from the HTTP/HTTPS server and returns a file-like object.
         """

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/extractors/local_file_extractor.py

@@ -0,0 +1,20 @@
+import logging
+from .base import Extractor
+from ..etl_configuration import ExtractorConfig
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, extractor_config: ExtractorConfig):
+        super().__init__(extractor_config)
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.cfg.source_uri, "r")
+            logging.info(f"Successfully opened file '{self.cfg.source_uri}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.cfg.source_uri}': {e}")
+            return None

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/factories.py

@@ -0,0 +1,23 @@
+from .extractors import HTTPExtractor, LocalFileExtractor
+from .transformers import JSONTransformer, CSVTransformer
+from .loaders import HydroServerLoader
+from .etl_configuration import ExtractorConfig, TransformerConfig, LoaderConfig
+
+EXTRACTORS = {"HTTP": HTTPExtractor, "local": LocalFileExtractor}
+TRANSFORMERS = {"JSON": JSONTransformer, "CSV": CSVTransformer}
+LOADERS = {"HydroServer": HydroServerLoader}
+
+
+def extractor_factory(settings: ExtractorConfig):
+    cls = EXTRACTORS[settings.type]
+    return cls(settings)
+
+
+def transformer_factory(settings: TransformerConfig):
+    cls = TRANSFORMERS[settings.type]
+    return cls(settings)
+
+
+def loader_factory(settings: LoaderConfig, auth_context, data_source_id: str):
+    cls = LOADERS[settings.type]
+    return cls(auth_context, data_source_id)

hydroserverpy-1.4.0b3/src/hydroserverpy/api/models/etl/loaders/__init__.py

@@ -0,0 +1,4 @@
+from .base import Loader
+from .hydroserver_loader import HydroServerLoader
+
+__all__ = ["Loader", "HydroServerLoader"]

{hydroserverpy-1.3.0b3/src/hydroserverpy → hydroserverpy-1.4.0b3/src/hydroserverpy/api/models}/etl/loaders/base.py

@@ -1,6 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import Dict
-import pandas as pd
 
 
 class Loader(ABC):