hydroserverpy 1.3.0b3__py3-none-any.whl → 1.4.0b3__py3-none-any.whl

hydroserverpy/__init__.py

@@ -1,9 +1,7 @@
 from .api.client import HydroServer
-from .etl.hydroserver_etl import HydroServerETL
 from .quality import HydroServerQualityControl
 
 __all__ = [
     "HydroServer",
     "HydroServerQualityControl",
-    "HydroServerETL",
 ]

hydroserverpy/api/models/etl/__init__.py

@@ -0,0 +1,26 @@
+from .extractors import Extractor, HTTPExtractor, LocalFileExtractor, FTPExtractor
+from .transformers import JSONTransformer, CSVTransformer, Transformer
+from .loaders import HydroServerLoader, Loader
+
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .orchestration_system import OrchestrationSystem
+from .data_source import DataSource
+
+__all__ = [
+    "CSVTransformer",
+    "JSONTransformer",
+    "LocalFileExtractor",
+    "FTPExtractor",
+    "HTTPExtractor",
+    "Extractor",
+    "Transformer",
+    "Loader",
+    "HydroServerLoader",
+    "EtlConfiguration",
+    "Schedule",
+    "Status",
+    "OrchestrationSystem",
+    "DataSource",
+]

hydroserverpy/api/models/etl/data_source.py

@@ -1,111 +1,146 @@
+from __future__ import annotations
+from datetime import datetime, timedelta, timezone
+from functools import cached_property
+import logging
 import uuid
-import tempfile
-import requests
-from typing import Union, ClassVar, Optional, TYPE_CHECKING, List
+from typing import ClassVar, TYPE_CHECKING, List, Optional, Union
+import croniter
+import pandas as pd
 from pydantic import Field
-from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
-from .orchestration_system import OrchestrationSystem
-from .orchestration_configuration import OrchestrationConfigurationFields
-from ..sta.datastream import Datastream
+
 from ..base import HydroServerBaseModel
+from ..sta.datastream import Datastream
+from .orchestration_system import OrchestrationSystem
+from .etl_configuration import EtlConfiguration
+from .schedule import Schedule
+from .status import Status
+from .factories import extractor_factory, transformer_factory, loader_factory
+from .loaders import HydroServerLoader
 
 if TYPE_CHECKING:
     from hydroserverpy import HydroServer
     from hydroserverpy.api.models import Workspace
 
 
-class DataSource(
-    HydroServerBaseModel, OrchestrationConfigurationFields
-):
+class DataSource(HydroServerBaseModel):
     name: str = Field(..., max_length=255)
-    settings: Optional[dict] = None
+    settings: EtlConfiguration
     orchestration_system_id: uuid.UUID
+    schedule: Schedule
+    status: Status
     workspace_id: uuid.UUID
 
     _editable_fields: ClassVar[set[str]] = {
-        "name", "settings", "interval", "interval_units", "crontab", "start_time", "end_time", "last_run_successful",
-        "last_run_message", "last_run", "next_run", "paused"
+        "name",
+        "settings",
+        "status",
+        "schedule",
+        "interval",
+        "interval_units",
+        "crontab",
+        "start_time",
+        "end_time",
+        "last_run_successful",
+        "last_run_message",
+        "last_run",
+        "next_run",
+        "paused",
     }
 
-    def __init__(self, client: "HydroServer", **data):
+    def __init__(self, client: HydroServer, **data):
         super().__init__(client=client, service=client.datasources, **data)
 
-        self._workspace = None
-        self._orchestration_system = None
-        self._datastreams = None
-
     @classmethod
     def get_route(cls):
         return "data-sources"
 
-    @property
-    def workspace(self) -> "Workspace":
-        """The workspace this data source belongs to."""
-
-        if self._workspace is None:
-            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
-
-        return self._workspace
-
-    @property
-    def orchestration_system(self) -> "OrchestrationSystem":
-        """The orchestration system that manages this data source."""
+    @cached_property
+    def workspace(self) -> Workspace:
+        return self.client.workspaces.get(uid=self.workspace_id)
 
-        if self._orchestration_system is None:
-            self._orchestration_system = self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
+    @cached_property
+    def orchestration_system(self) -> OrchestrationSystem:
+        return self.client.orchestrationsystems.get(uid=self.orchestration_system_id)
 
-        return self._orchestration_system
-
-    @property
-    def datastreams(self) -> List["Datastream"]:
-        """The datastreams this data source provides data for."""
-
-        if self._datastreams is None:
-            self._datastreams = self.client.datastreams.list(data_source=self.uid, fetch_all=True).items
-
-        return self._datastreams
+    @cached_property
+    def datastreams(self) -> List[Datastream]:
+        return self.client.datastreams.list(data_source=self.uid, fetch_all=True).items
 
+    # TODO: Add functions like add_payload, add_mapping, etc. and don't allow the user to manually
+    # link or unlink datastreams - handle that automatically.
     def add_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
         """Add a datastream to this data source."""
 
-        self.client.datasources.add_datastream(
-            uid=self.uid, datastream=datastream
-        )
+        self.client.datasources.add_datastream(uid=self.uid, datastream=datastream)
 
     def remove_datastream(self, datastream: Union["Datastream", uuid.UUID, str]):
         """Remove a datastream from this data source."""
 
-        self.client.datasources.remove_datastream(
-            uid=self.uid, datastream=datastream
+        self.client.datasources.remove_datastream(uid=self.uid, datastream=datastream)
+
+    def _next_run(self) -> Optional[str]:
+        now = datetime.now(timezone.utc)
+        if cron := self.schedule.crontab:
+            return croniter.croniter(cron, now).get_next(datetime).isoformat()
+        if iv := self.schedule.interval:
+            unit = self.schedule.interval_units or "minutes"
+            return (now + timedelta(**{unit: iv})).isoformat()
+        return None
+
+    def _update_status(self, loader: HydroServerLoader, success: bool, msg: str):
+        short_msg = msg if len(msg) <= 255 else msg[:252] + "…"
+        loader.client.datasources.update(
+            uid=self.uid,
+            last_run=datetime.now(timezone.utc).isoformat(),
+            last_run_successful=success,
+            last_run_message=short_msg,
+            next_run=self._next_run(),
         )
 
-    # TODO: Replace with ETL module.
-    def load_data(self):
-        """Load data for this data source."""
+    def is_empty(self, data):
+        if data is None:
+            return True
+        if isinstance(data, pd.DataFrame) and data.empty:
+            return True
+        return False
 
-        if self.paused is True:
+    def load_data(self, payload_name: str = None):
+        """Load data for this data source."""
+        if self.status.paused is True:
             return
 
-        if self.settings["extractor"]["type"] == "local":
-            with open(self.settings["extractor"]["sourceUri"]) as data_file:
-                loader = HydroServerETLCSV(
-                    self.client, data_file=data_file, data_source=self
-                )
-                loader.run()
-        elif self.settings["extractor"]["type"] == "HTTP":
-            with tempfile.NamedTemporaryFile(mode="w+") as temp_file:
-                response = requests.get(
-                    self.settings["extractor"]["sourceUri"],
-                    stream=True,
-                    timeout=60,
+        if payload_name:
+            self.load_data_for_payload(payload_name)
+        else:
+            for p in self.settings.payloads:
+                self.load_data_for_payload(p.name)
+
+    def load_data_for_payload(self, payload_name: str):
+        payload = next(p for p in self.settings.payloads if p.name == payload_name)
+
+        extractor_cls = extractor_factory(self.settings.extractor)
+        transformer_cls = transformer_factory(self.settings.transformer)
+        loader_cls = loader_factory(self.settings.loader, self.client, self.uid)
+
+        try:
+            logging.info("Starting extract")
+            data = extractor_cls.extract(payload, loader_cls)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the extractor"
                 )
-                response.raise_for_status()
-                chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
-                for chunk in response.iter_content(chunk_size=chunk_size):
-                    if chunk:
-                        temp_file.write(chunk.decode("utf-8"))
-                temp_file.seek(0)
-                loader = HydroServerETLCSV(
-                    self.client, data_file=temp_file, data_source=self
+                return
+
+            logging.info("Starting transform")
+            data = transformer_cls.transform(data, payload.mappings)
+            if self.is_empty(data):
+                self._update_status(
+                    loader_cls, True, "No data returned from the transformer"
                 )
-                loader.run()
+                return
+
+            logging.info("Starting load")
+            loader_cls.load(data, payload)
+            self._update_status(loader_cls, True, "OK")
+        except Exception as e:
+            self._update_status(loader_cls, False, str(e))

hydroserverpy/api/models/etl/etl_configuration.py

@@ -0,0 +1,224 @@
+from typing import Annotated, Dict, List, Literal, Optional, Union
+from pydantic import BaseModel, Field, field_validator
+from enum import Enum
+
+WorkflowType = Literal["ETL", "Aggregation", "Virtual", "SDL"]
+CSVDelimiterType = Literal[",", "|", "\t", ";", " "]
+ExtractorType = Literal["HTTP", "local"]
+TransformerType = Literal["JSON", "CSV"]
+LoaderType = Literal["HydroServer"]
+IdentifierType = Literal["name", "index"]
+RunTimeValue = Literal["jobExecutionTime", "latestObservationTimestamp"]
+
+
+class FixedOffsetTimezone(str, Enum):
+    UTC_MINUS_1200 = "-1200"
+    UTC_MINUS_1100 = "-1100"
+    UTC_MINUS_1000 = "-1000"
+    UTC_MINUS_0900 = "-0900"
+    UTC_MINUS_0800 = "-0800"
+    UTC_MINUS_0700 = "-0700"
+    UTC_MINUS_0600 = "-0600"
+    UTC_MINUS_0500 = "-0500"
+    UTC_MINUS_0430 = "-0430"
+    UTC_MINUS_0400 = "-0400"
+    UTC_MINUS_0330 = "-0330"
+    UTC_MINUS_0300 = "-0300"
+    UTC_MINUS_0200 = "-0200"
+    UTC_MINUS_0100 = "-0100"
+    UTC_PLUS_0000 = "+0000"
+    UTC_PLUS_0100 = "+0100"
+    UTC_PLUS_0200 = "+0200"
+    UTC_PLUS_0300 = "+0300"
+    UTC_PLUS_0330 = "+0330"
+    UTC_PLUS_0400 = "+0400"
+    UTC_PLUS_0430 = "+0430"
+    UTC_PLUS_0500 = "+0500"
+    UTC_PLUS_0530 = "+0530"
+    UTC_PLUS_0545 = "+0545"
+    UTC_PLUS_0600 = "+0600"
+    UTC_PLUS_0630 = "+0630"
+    UTC_PLUS_0700 = "+0700"
+    UTC_PLUS_0800 = "+0800"
+    UTC_PLUS_0845 = "+0845"
+    UTC_PLUS_0900 = "+0900"
+    UTC_PLUS_0930 = "+0930"
+    UTC_PLUS_1000 = "+1000"
+    UTC_PLUS_1030 = "+1030"
+    UTC_PLUS_1100 = "+1100"
+    UTC_PLUS_1130 = "+1130"
+    UTC_PLUS_1200 = "+1200"
+    UTC_PLUS_1245 = "+1245"
+    UTC_PLUS_1300 = "+1300"
+    UTC_PLUS_1400 = "+1400"
+
+
+class TimestampFormat(str, Enum):
+    ISO8601 = "ISO8601"
+    naive = "naive"
+    custom = "custom"
+
+
+class TimezoneMode(str, Enum):
+    utc = "utc"  # always UTC
+    daylightSavings = "daylightSavings"  # IANA / DST-aware
+    fixedOffset = "fixedOffset"  # constant offset
+    embeddedOffset = "embeddedOffset"  # offset in ISO string
+
+
+class Timestamp(BaseModel):
+    key: Optional[str] = None
+    format: TimestampFormat
+    custom_format: Optional[str] = Field(None, alias="customFormat")
+    timezone_mode: TimezoneMode = Field(..., alias="timezoneMode")
+    timezone: Optional[Union[FixedOffsetTimezone, str]] = Field(None, alias="timezone")
+
+    class Config:
+        allow_population_by_field_name = True
+
+    @field_validator("timezone")
+    def check_timezone(cls, timezone_value, info):
+        mode = info.data.get("timezone_mode")
+        if mode == TimezoneMode.fixedOffset and timezone_value is None:
+            raise ValueError("`timezone` must be set when timezoneMode is fixedOffset")
+        return timezone_value
+
+
+class PerPayloadPlaceholder(BaseModel):
+    name: str
+    type: Literal["perPayload"]
+
+
+class RunTimePlaceholder(BaseModel):
+    name: str
+    type: Literal["runTime"]
+    run_time_value: RunTimeValue = Field(..., alias="runTimeValue")
+    timestamp: Timestamp
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+PlaceholderVariable = Annotated[
+    Union[PerPayloadPlaceholder, RunTimePlaceholder],
+    Field(discriminator="type"),
+]
+
+
+class BaseExtractor(BaseModel):
+    type: ExtractorType
+    source_uri: str = Field(..., alias="sourceUri")
+    placeholder_variables: Optional[List[PlaceholderVariable]] = Field(
+        default_factory=list,
+        alias="placeholderVariables",
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class HTTPExtractor(BaseExtractor):
+    type: Literal["HTTP"]
+
+
+class LocalFileExtractor(BaseExtractor):
+    type: Literal["local"]
+
+
+ExtractorConfig = Annotated[
+    Union[HTTPExtractor, LocalFileExtractor], Field(discriminator="type")
+]
+
+
+class BaseTransformer(BaseModel):
+    type: TransformerType
+    timestamp: Timestamp
+
+
+class JSONTransformer(BaseTransformer):
+    type: Literal["JSON"]
+    jmespath: str = Field(..., alias="JMESPath")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class CSVTransformer(BaseTransformer):
+    type: Literal["CSV"]
+    header_row: Optional[int] = Field(..., alias="headerRow")
+    data_start_row: int = Field(..., alias="dataStartRow")
+    delimiter: CSVDelimiterType
+    identifier_type: IdentifierType = Field(..., alias="identifierType")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+TransformerConfig = Union[JSONTransformer, CSVTransformer]
+
+
+class BaseLoaderConfig(BaseModel):
+    type: LoaderType
+
+
+class HydroServerLoaderConfig(BaseLoaderConfig):
+    type: Literal["HydroServer"]
+
+
+LoaderConfig = HydroServerLoaderConfig
+
+
+class ExpressionDataTransformation(BaseModel):
+    type: Literal["expression"]
+    expression: str
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class LookupTableDataTransformation(BaseModel):
+    type: Literal["lookup"]
+    lookup_table_id: str = Field(..., alias="lookupTableId")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+DataTransformation = Union[ExpressionDataTransformation, LookupTableDataTransformation]
+
+
+class MappingPath(BaseModel):
+    target_identifier: Union[str, int] = Field(..., alias="targetIdentifier")
+    data_transformations: List[DataTransformation] = Field(
+        default_factory=list, alias="dataTransformations"
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class SourceTargetMapping(BaseModel):
+    source_identifier: Union[str, int] = Field(..., alias="sourceIdentifier")
+    paths: List[MappingPath] = Field(default_factory=list)
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Payload(BaseModel):
+    name: str = ""
+    mappings: List[SourceTargetMapping] = Field(default_factory=list)
+    extractor_variables: Dict[str, str] = Field(
+        default_factory=dict, alias="extractorVariables"
+    )
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class EtlConfiguration(BaseModel):
+    type: WorkflowType
+    extractor: ExtractorConfig
+    transformer: TransformerConfig
+    loader: LoaderConfig
+    payloads: List[Payload]

hydroserverpy/api/models/etl/extractors/__init__.py

@@ -0,0 +1,6 @@
+from .base import Extractor
+from .ftp_extractor import FTPExtractor
+from .http_extractor import HTTPExtractor
+from .local_file_extractor import LocalFileExtractor
+
+__all__ = ["Extractor", "HTTPExtractor", "LocalFileExtractor", "FTPExtractor"]

hydroserverpy/{etl → api/models/etl}/extractors/base.py

@@ -2,49 +2,46 @@ from abc import abstractmethod
 import logging
 import pandas as pd
 from datetime import datetime
-
-from hydroserverpy.etl.timestamp_parser import TimestampParser
+from ..etl_configuration import ExtractorConfig, Payload
+from ..timestamp_parser import TimestampParser
 
 
 class Extractor:
-    def __init__(self, settings: dict):
-        self.settings = settings
-        self.source_uri = settings["sourceUri"]
+    def __init__(self, extractor_config: ExtractorConfig):
+        self.cfg = extractor_config
 
-    def resolve_placeholder_variables(self, payload, loader):
+    def resolve_placeholder_variables(self, payload: Payload, loader):
         logging.info(f"Creating runtime variables...")
         filled = {}
-        for var in self.settings.get("placeholderVariables", []):
-            name = var["name"]
-            var_type = var.get("type", None)
+        for placeholder in self.cfg.placeholder_variables:
+            name = placeholder.name
 
-            if var_type == "runTime":
+            if placeholder.type == "runTime":
                 logging.info(f"Resolving runtime var: {name}")
-                if var.get("runTimeValue", None) == "latestObservationTimestamp":
+                if placeholder.run_time_value == "latestObservationTimestamp":
                     value = loader.earliest_begin_date(payload)
-                elif var.get("runTimeValue", None) == "jobExecutionTime":
+                elif placeholder.run_time_value == "jobExecutionTime":
                     value = pd.Timestamp.now(tz="UTC")
-            elif var_type == "perPayload":
+            elif placeholder.type == "perPayload":
                 logging.info(f"Resolving payload var: {name}")
-                payload_vars = payload.get("extractorVariables", {})
-                if name not in payload_vars:
+                if name not in payload.extractor_variables:
                     raise KeyError(f"Missing per-payload variable '{name}'")
-                value = payload_vars[name]
+                value = payload.extractor_variables[name]
             else:
                 continue
 
             if isinstance(value, (datetime, pd.Timestamp)):
-                parser = TimestampParser(var["timestamp"])
+                parser = TimestampParser(placeholder.timestamp)
                 value = parser.utc_to_string(value)
 
             filled[name] = value
         if not filled:
-            return self.source_uri
+            return self.cfg.source_uri
         return self.format_uri(filled)
 
     def format_uri(self, placeholder_variables):
         try:
-            uri = self.source_uri.format(**placeholder_variables)
+            uri = self.cfg.source_uri.format(**placeholder_variables)
         except KeyError as e:
             missing_key = e.args[0]
             raise KeyError(f"Missing placeholder variable: {missing_key}")

hydroserverpy/{etl → api/models/etl}/extractors/http_extractor.py

@@ -1,14 +1,16 @@
 import logging
 import requests
 from io import BytesIO
-from .base import Extractor
+
+from ..etl_configuration import Payload
+from .base import Extractor, ExtractorConfig
 
 
 class HTTPExtractor(Extractor):
-    def __init__(self, settings: dict):
+    def __init__(self, settings: ExtractorConfig):
         super().__init__(settings)
 
-    def extract(self, payload, loader=None):
+    def extract(self, payload: Payload, loader=None):
         """
         Downloads the file from the HTTP/HTTPS server and returns a file-like object.
         """

hydroserverpy/api/models/etl/extractors/local_file_extractor.py

@@ -0,0 +1,20 @@
+import logging
+from .base import Extractor
+from ..etl_configuration import ExtractorConfig
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, extractor_config: ExtractorConfig):
+        super().__init__(extractor_config)
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.cfg.source_uri, "r")
+            logging.info(f"Successfully opened file '{self.cfg.source_uri}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.cfg.source_uri}': {e}")
+            return None

hydroserverpy/api/models/etl/factories.py

@@ -0,0 +1,23 @@
+from .extractors import HTTPExtractor, LocalFileExtractor
+from .transformers import JSONTransformer, CSVTransformer
+from .loaders import HydroServerLoader
+from .etl_configuration import ExtractorConfig, TransformerConfig, LoaderConfig
+
+EXTRACTORS = {"HTTP": HTTPExtractor, "local": LocalFileExtractor}
+TRANSFORMERS = {"JSON": JSONTransformer, "CSV": CSVTransformer}
+LOADERS = {"HydroServer": HydroServerLoader}
+
+
+def extractor_factory(settings: ExtractorConfig):
+    cls = EXTRACTORS[settings.type]
+    return cls(settings)
+
+
+def transformer_factory(settings: TransformerConfig):
+    cls = TRANSFORMERS[settings.type]
+    return cls(settings)
+
+
+def loader_factory(settings: LoaderConfig, auth_context, data_source_id: str):
+    cls = LOADERS[settings.type]
+    return cls(auth_context, data_source_id)

hydroserverpy/api/models/etl/loaders/__init__.py

@@ -0,0 +1,4 @@
+from .base import Loader
+from .hydroserver_loader import HydroServerLoader
+
+__all__ = ["Loader", "HydroServerLoader"]

hydroserverpy/{etl → api/models/etl}/loaders/base.py

@@ -1,6 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import Dict
-import pandas as pd
 
 
 class Loader(ABC):