hydroserverpy 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

hydroserverpy/etl/hydroserver_etl.py

@@ -0,0 +1,40 @@
+import logging
+import pandas as pd
+
+
+class HydroServerETL:
+    def __init__(self, extractor, transformer, loader, source_target_map):
+        self.extractor = extractor
+        self.transformer = transformer
+        self.loader = loader
+        self.source_target_map = source_target_map
+
+    def run(self):
+        """
+        Extracts, transforms, and loads data as defined by the class parameters.
+        """
+
+        # Step 1: Get Target System data requirements from the Loader & prepare parameters for the Extractor
+        data_requirements = self.loader.get_data_requirements(self.source_target_map)
+        self.extractor.prepare_params(data_requirements)
+
+        # Step 2: Extract
+        data = self.extractor.extract()
+        if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+            logging.warning(f"No data was returned from the extractor. Ending ETL run.")
+            return
+        else:
+            logging.info(f"Successfully extracted data.")
+
+        # Step 3: Transform
+        if self.transformer:
+            data = self.transformer.transform(data)
+            if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+                logging.warning(f"No data returned from the transformer. Ending run.")
+                return
+            else:
+                logging.info(f"Successfully transformed data. {data}")
+
+        # Step 4: Load
+        self.loader.load(data, self.source_target_map)
+        logging.info("Successfully loaded data.")

hydroserverpy/etl/loaders/base.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+import pandas as pd
+
+
+class Loader(ABC):
+    @abstractmethod
+    def load(self, *args, **kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def get_data_requirements(self, df: pd.DataFrame) -> Dict[str, pd.Timestamp]:
+        pass

hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -0,0 +1,68 @@
+from hydroserverpy.core.service import HydroServer
+from typing import Dict, Optional
+from .base import Loader
+import logging
+import pandas as pd
+
+
+class HydroServerLoader(HydroServer, Loader):
+    """
+    A class that extends the HydroServer client with ETL-specific functionalities.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+        api_route: str = "api",
+    ):
+        super().__init__(host, username, password, apikey, api_route)
+
+    def load(self, data: pd.DataFrame, source_target_map) -> None:
+        """
+        Load observations from a DataFrame to the HydroServer.
+
+        :param data: A Pandas DataFrame where each column corresponds to a datastream.
+        """
+        data_requirements = self.get_data_requirements(source_target_map)
+        for ds_id in data.columns:
+            if ds_id == "timestamp":
+                continue
+
+            df = data[["timestamp", ds_id]].copy()
+            df.rename(columns={ds_id: "value"}, inplace=True)
+            df.dropna(subset=["value"], inplace=True)
+
+            phenomenon_end_time = data_requirements[ds_id]["start_time"]
+            if phenomenon_end_time:
+                df = df[df["timestamp"] > phenomenon_end_time]
+            if df.empty:
+                logging.warning(
+                    f"No new data to upload for datastream {ds_id}. Skipping."
+                )
+                continue
+            self.datastreams.load_observations(uid=ds_id, observations=df)
+
+    def get_data_requirements(
+        self, source_target_map
+    ) -> Dict[str, Dict[str, pd.Timestamp]]:
+        """
+        Each target system needs to be able to answer the question: 'What data do you need?'
+        and return a time range for each target time series. Usually the answer will be
+        'anything newer than my most recent observation'.
+        """
+        data_requirements = {}
+        for ds_id in source_target_map.values():
+            datastream = self.datastreams.get(uid=ds_id)
+            if not datastream:
+                message = "Couldn't fetch target datastream. ETL process aborted."
+                logging.error(message)
+                raise message
+            start_time = pd.Timestamp(
+                datastream.phenomenon_end_time or "1970-01-01T00:00:00Z"
+            )
+            end_time = pd.Timestamp.now()
+            data_requirements[ds_id] = {"start_time": start_time, "end_time": end_time}
+        return data_requirements

hydroserverpy/etl/transformers/base.py

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+import logging
+import pandas as pd
+
+
+class Transformer(ABC):
+    @abstractmethod
+    def transform(self, *args, **kwargs) -> None:
+        pass
+
+    @property
+    def needs_datastreams(self) -> bool:
+        return False
+
+    @staticmethod
+    def standardize_dataframe(
+        df,
+        datastream_ids,
+        timestamp_column: str = "timestamp",
+        timestamp_format: str = "ISO8601",
+    ):
+        df.rename(
+            columns={timestamp_column: "timestamp", **datastream_ids},
+            inplace=True,
+        )
+
+        # Verify timestamp column is present in the DataFrame
+        if "timestamp" not in df.columns:
+            message = f"Timestamp column '{timestamp_column}' not found in data."
+            logging.error(message)
+            raise ValueError(message)
+
+        # Verify that all datastream_ids are present in the DataFrame
+        expected_columns = set(datastream_ids.values())
+        actual_columns = set(df.columns)
+        missing_datastream_ids = expected_columns - actual_columns
+
+        if missing_datastream_ids:
+            raise ValueError(
+                "The following datastream IDs are specified in the config file but their related keys could not be "
+                f"found in the source system's extracted data: {missing_datastream_ids}"
+            )
+
+        # Keep only 'timestamp' and datastream_id columns
+        columns_to_keep = ["timestamp"] + list(expected_columns)
+        df = df[columns_to_keep]
+
+        # Convert timestamp column to datetime if not already
+        if not pd.api.types.is_datetime64_any_dtype(df["timestamp"]):
+            df["timestamp"] = pd.to_datetime(df["timestamp"], format=timestamp_format)
+
+        return df

hydroserverpy/etl/transformers/csv_transformer.py

@@ -0,0 +1,88 @@
+import logging
+import pandas as pd
+from typing import Dict, Optional, Union
+from .base import Transformer
+
+
+class CSVTransformer(Transformer):
+    def __init__(
+        self,
+        header_row: Optional[int],
+        data_start_row: int,
+        timestamp_column: Union[str, int],
+        datastream_ids: Dict[Union[str, int], str],
+        delimiter: Optional[str] = ",",
+        timestamp_format: Optional[str] = "ISO8601",
+    ):
+        # Pandas is zero-based while CSV is one-based so convert
+        self.header_row = None if header_row is None else header_row - 1
+        self.data_start_row = data_start_row - 1
+        self.timestamp_column = self.convert_to_zero_based(timestamp_column)
+        self.datastream_ids = datastream_ids
+        self.timestamp_format = timestamp_format
+        self.delimiter = delimiter
+
+    def transform(self, data_file) -> Union[pd.DataFrame, None]:
+        """
+        Transforms a CSV file-like object into a Pandas DataFrame where the column
+        names are replaced with their target datastream ids.
+
+        Parameters:
+            data_file: File-like object containing CSV data.
+        Returns:
+            observations_map (dict): Dict mapping datastream IDs to pandas DataFrames.
+        """
+
+        try:
+            df = pd.read_csv(
+                data_file,
+                delimiter=self.delimiter,
+                header=self.header_row,
+                parse_dates=[self.timestamp_column],
+                date_format=self.timestamp_format,
+                skiprows=self.calculate_skiprows(),
+                usecols=[self.timestamp_column] + list(self.datastream_ids.keys()),
+            )
+        except Exception as e:
+            logging.error(f"Error reading CSV data: {e}")
+            return None
+
+        if self.header_row is None:
+            df.columns = list(range(1, len(df.columns) + 1))
+
+        return self.standardize_dataframe(
+            df, self.datastream_ids, self.timestamp_column, self.timestamp_format
+        )
+
+    def calculate_skiprows(self):
+        """
+        Calculates the skiprows parameter for pd.read_csv.
+
+        Returns:
+            skiprows (list or None): List of row indices to skip, or None if no rows need to be skipped.
+        Raises:
+            ValueError: If header_row is not compatible with data_start_row.
+        """
+        if self.data_start_row == 0:
+            if self.header_row is not None:
+                # Cannot have a header row if data starts at the first row
+                raise ValueError(
+                    "header_row must be None when data_start_row is 1 (first row)"
+                )
+            return None  # No rows to skip
+
+        skiprows = list(range(self.data_start_row))
+
+        if self.header_row is not None:
+            if self.header_row >= self.data_start_row:
+                raise ValueError("header_row must be less than data_start_row")
+            if self.header_row in skiprows:
+                # Do not skip the header row
+                skiprows.remove(self.header_row)
+        return skiprows
+
+    @staticmethod
+    def convert_to_zero_based(index: Union[str, int]) -> Union[str, int]:
+        if isinstance(index, int):
+            return index - 1
+        return index

hydroserverpy/etl/transformers/json_transformer.py

@@ -0,0 +1,62 @@
+import logging
+import pandas as pd
+from typing import Dict, Optional, Any, List
+from .base import Transformer
+import json
+import jmespath
+
+
+class JSONTransformer(Transformer):
+    def __init__(
+        self,
+        query_string: str,
+        datastream_ids: Dict[str, str],
+        timestamp_format: Optional[str] = "ISO8601",
+    ):
+        """
+        Initializes the JSONTransformer.
+
+        Parameters:
+            query_string (str): JMESPath to the data array containing time series data.
+            Since JMESPath can natively rename column names, the assumption is the timestamp column
+            is always named 'timestamp' or converted to 'timestamp' in the JMESPath query.
+            datastream_ids (dict): Mapping from JSON field names to datastream IDs.
+            timestamp_format (str, optional): The format of the timestamp, if it needs special parsing.
+        """
+        self.query_string = query_string
+        self.datastream_ids = datastream_ids
+        self.timestamp_format = timestamp_format
+
+    def transform(self, data_file):
+        """
+        Transforms a JSON file-like object into the standard Pandas dataframe format.
+        Since JMESPath can natively rename column names, the assumption is the timestamp column
+        is always named 'timestamp' for JSON data or converted to 'timestamp' in the JMESPath query.
+
+        Parameters:
+            data_file: File-like object containing JSON data.
+
+        Returns:
+            pd.DataFrame: pandas DataFrames in the format pd.Timestamp, datastream_id_1, datastream_id_2, ...
+        """
+        json_data = json.load(data_file)
+        data_points = self.extract_data_points(json_data)
+        if not data_points:
+            logging.warning("No data points found in the JSON data.")
+            return None
+
+        df = pd.DataFrame(data_points)
+
+        return self.standardize_dataframe(
+            df,
+            self.datastream_ids,
+            timestamp_format=self.timestamp_format,
+        )
+
+    def extract_data_points(self, json_data: Any) -> Optional[List[dict]]:
+        """Extracts data points from the JSON data using the data_path."""
+        data_points = jmespath.search(self.query_string, json_data)
+
+        if isinstance(data_points, dict):
+            data_points = [data_points]
+        return data_points

hydroserverpy/etl/types.py

@@ -0,0 +1,7 @@
+from typing import TypedDict
+import pandas as pd
+
+
+class TimeRange(TypedDict):
+    start_time: pd.Timestamp
+    end_time: pd.Timestamp

hydroserverpy/{etl/service.py → etl_csv/hydroserver_etl_csv.py}

@@ -7,22 +7,28 @@ from requests import HTTPError
 from datetime import datetime, timezone, timedelta
 from dateutil.parser import isoparse
 from .exceptions import HeaderParsingError, TimestampParsingError
+import warnings
 
 if TYPE_CHECKING:
     from ..core.schemas import DataSource
 
-logger = logging.getLogger('hydroserver_etl')
+logger = logging.getLogger("hydroserver_etl")
 logger.addHandler(logging.NullHandler())
 
 
-class HydroServerETL:
+class HydroServerETLCSV:
 
     def __init__(
-            self,
-            service,
-            data_file: IO[str],
-            data_source: 'DataSource',
+        self,
+        service,
+        data_file: IO[str],
+        data_source: "DataSource",
     ):
+        warnings.warn(
+            "HydroServerETLCSV is deprecated and will be removed in a future version. "
+            "Please use the new HydroServerETL class.",
+            DeprecationWarning,
+        )
         self._service = service
         self._data_file = data_file
         self._data_source = data_source
@@ -66,12 +72,12 @@ class HydroServerETL:
                     self._failed_datastreams.extend(self._post_observations())
 
         except HeaderParsingError as e:
-            self._message = f'Failed to parse header for {self._data_source.name} with error: {str(e)}'
+            self._message = f"Failed to parse header for {self._data_source.name} with error: {str(e)}"
             logger.error(self._message)
             self._file_header_error = True
 
         except TimestampParsingError as e:
-            self._message = f'Failed to parse one or more timestamps for {self._data_source.name} with error: {str(e)}'
+            self._message = f"Failed to parse one or more timestamps for {self._data_source.name} with error: {str(e)}"
             logger.error(self._message)
             self._file_timestamp_error = True
 
@@ -79,7 +85,7 @@ class HydroServerETL:
         self._failed_datastreams.extend(self._post_observations())
 
         if not self._message and len(self._failed_datastreams) > 0:
-            self._message = f'One or more datastreams failed to sync with HydroServer for {self._data_source.name}.'
+            self._message = f"One or more datastreams failed to sync with HydroServer for {self._data_source.name}."
 
         self._update_data_source()
 
@@ -99,7 +105,8 @@ class HydroServerETL:
         """
 
         if index == self._data_source.header_row or (
-                index == self._data_source.data_start_row and self._timestamp_column_index is None
+            index == self._data_source.data_start_row
+            and self._timestamp_column_index is None
         ):
             self._parse_file_header(row)
 
@@ -110,18 +117,29 @@ class HydroServerETL:
 
         for datastream in self._datastreams.values():
             if str(datastream.uid) not in self._datastream_start_row_indexes.keys():
-                if not datastream.phenomenon_end_time or timestamp > datastream.phenomenon_end_time:
+                if (
+                    not datastream.phenomenon_end_time
+                    or timestamp > datastream.phenomenon_end_time
+                ):
                     self._datastream_start_row_indexes[str(datastream.uid)] = index
 
-            if str(datastream.uid) in self._datastream_start_row_indexes.keys() \
-                    and self._datastream_start_row_indexes[str(datastream.uid)] <= index:
+            if (
+                str(datastream.uid) in self._datastream_start_row_indexes.keys()
+                and self._datastream_start_row_indexes[str(datastream.uid)] <= index
+            ):
                 if str(datastream.uid) not in self._observations.keys():
                     self._observations[str(datastream.uid)] = []
 
-                self._observations[str(datastream.uid)].append({
-                    'phenomenon_time': timestamp,
-                    'result': row[self._datastream_column_indexes[datastream.data_source_column]]
-                })
+                self._observations[str(datastream.uid)].append(
+                    {
+                        "phenomenon_time": timestamp,
+                        "result": row[
+                            self._datastream_column_indexes[
+                                datastream.data_source_column
+                            ]
+                        ],
+                    }
+                )
 
     def _parse_file_header(self, row: List[str]) -> None:
         """
@@ -136,22 +154,29 @@ class HydroServerETL:
         """
 
         try:
-            self._timestamp_column_index = row.index(self._data_source.timestamp_column) \
-                if isinstance(self._data_source.timestamp_column, str) \
+            self._timestamp_column_index = (
+                row.index(self._data_source.timestamp_column)
+                if isinstance(self._data_source.timestamp_column, str)
                 else int(self._data_source.timestamp_column) - 1
+            )
             if self._timestamp_column_index > len(row):
                 raise ValueError
             self._datastream_column_indexes = {
-                datastream.data_source_column: row.index(datastream.data_source_column)
-                if not datastream.data_source_column.isdigit()
-                else int(datastream.data_source_column) - 1
+                datastream.data_source_column: (
+                    row.index(datastream.data_source_column)
+                    if not datastream.data_source_column.isdigit()
+                    else int(datastream.data_source_column) - 1
+                )
                 for datastream in self._datastreams.values()
             }
-            if len(self._datastream_column_indexes.values()) > 0 and \
-                    max(self._datastream_column_indexes.values()) > len(row):
+            if len(self._datastream_column_indexes.values()) > 0 and max(
+                self._datastream_column_indexes.values()
+            ) > len(row):
                 raise ValueError
         except ValueError as e:
-            logger.error(f'Failed to load data from data source: "{self._data_source.name}"')
+            logger.error(
+                f'Failed to load data from data source: "{self._data_source.name}"'
+            )
             raise HeaderParsingError(str(e)) from e
 
     def _parse_row_timestamp(self, row: List[str]) -> datetime:
@@ -164,32 +189,36 @@ class HydroServerETL:
         """
 
         try:
-            if self._data_source.timestamp_format == 'iso' or self._data_source.timestamp_format is None:
-                timestamp = isoparse(
-                    row[self._timestamp_column_index]
-                )
+            if (
+                self._data_source.timestamp_format == "iso"
+                or self._data_source.timestamp_format is None
+            ):
+                timestamp = isoparse(row[self._timestamp_column_index])
             else:
                 timestamp = datetime.strptime(
                     row[self._timestamp_column_index],
-                    self._data_source.timestamp_format
+                    self._data_source.timestamp_format,
                 )
         except ValueError as e:
             raise TimestampParsingError(str(e)) from e
 
         if timestamp.tzinfo is None:
             if not self._data_source.timestamp_offset:
-                timestamp = timestamp.replace(
-                    tzinfo=timezone.utc
-                )
+                timestamp = timestamp.replace(tzinfo=timezone.utc)
             else:
                 try:
                     timestamp = timestamp.replace(
                         tzinfo=datetime.strptime(
-                            self._data_source.timestamp_offset[:-2] + ':' + self._data_source.timestamp_offset[3:], '%z'
+                            self._data_source.timestamp_offset[:-2]
+                            + ":"
+                            + self._data_source.timestamp_offset[3:],
+                            "%z",
                         ).tzinfo
                     )
                 except ValueError as e:
-                    logger.error(f'Failed to load data from data source: "{self._data_source.name}"')
+                    logger.error(
+                        f'Failed to load data from data source: "{self._data_source.name}"'
+                    )
                     raise TimestampParsingError(str(e)) from e
 
         return timestamp
@@ -213,15 +242,18 @@ class HydroServerETL:
             if datastream_id not in self._failed_datastreams and len(observations) > 0:
 
                 logger.info(
-                    f'Loading observations from ' +
-                    f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to ' +
-                    f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: ' +
-                    f'{str(datastream_id)} in data source "{self._data_source.name}".'
+                    f"Loading observations from "
+                    + f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to '
+                    + f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: '
+                    + f'{str(datastream_id)} in data source "{self._data_source.name}".'
                 )
 
                 observations_df = pd.DataFrame(
-                    [[observation['phenomenon_time'], observation['result']] for observation in observations],
-                    columns=['timestamp', 'value']
+                    [
+                        [observation["phenomenon_time"], observation["result"]]
+                        for observation in observations
+                    ],
+                    columns=["timestamp", "value"],
                 )
 
                 try:
@@ -233,17 +265,18 @@ class HydroServerETL:
                     failed_datastreams.append(datastream_id)
 
                 if not self._last_loaded_timestamp or (
-                        observations[-1]['phenomenon_time'] and observations[-1]['phenomenon_time'] >
-                        self._last_loaded_timestamp
+                    observations[-1]["phenomenon_time"]
+                    and observations[-1]["phenomenon_time"]
+                    > self._last_loaded_timestamp
                 ):
-                    self._last_loaded_timestamp = observations[-1]['phenomenon_time']
+                    self._last_loaded_timestamp = observations[-1]["phenomenon_time"]
             elif datastream_id in self._failed_datastreams:
                 logger.info(
-                    f'Skipping observations POST request from ' +
-                    f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to ' +
-                    f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: ' +
-                    f'{str(datastream_id)} in data source "{self._data_source.name}",' +
-                    f'due to previous failed POST request.'
+                    f"Skipping observations POST request from "
+                    + f'{observations[0]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} to '
+                    + f'{observations[-1]["phenomenon_time"].strftime("%Y-%m-%dT%H:%M:%S%z")} for datastream: '
+                    + f'{str(datastream_id)} in data source "{self._data_source.name}",'
+                    + f"due to previous failed POST request."
                 )
 
         self._observations = {}
@@ -260,10 +293,12 @@ class HydroServerETL:
 
         if self._data_source.crontab is not None:
             next_sync = croniter.croniter(
-                self._data_source.crontab,
-                datetime.now()
+                self._data_source.crontab, datetime.now()
             ).get_next(datetime)
-        elif self._data_source.interval is not None and self._data_source.interval_units is not None:
+        elif (
+            self._data_source.interval is not None
+            and self._data_source.interval_units is not None
+        ):
             next_sync = datetime.now() + timedelta(
                 **{self._data_source.interval_units: self._data_source.interval}
             )
@@ -272,8 +307,11 @@ class HydroServerETL:
 
         self._data_source.data_source_thru = self._last_loaded_timestamp
         self._data_source.last_sync_successful = (
-            True if not self._file_timestamp_error and not self._file_header_error
-            and len(self._failed_datastreams) == 0 else False
+            True
+            if not self._file_timestamp_error
+            and not self._file_header_error
+            and len(self._failed_datastreams) == 0
+            else False
         )
         self._data_source.last_sync_message = self._message
         self._data_source.last_synced = datetime.now(timezone.utc)