hydroserverpy 0.2.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

hydroserverpy/etl/extractors/http_extractor.py

@@ -0,0 +1,84 @@
+import logging
+from hydroserverpy.etl.types import TimeRange
+import requests
+from io import BytesIO
+from typing import Dict
+from .base import Extractor
+
+
+class HTTPExtractor(Extractor):
+    def __init__(
+        self,
+        url: str,
+        url_variables: dict = None,
+        params: dict = None,
+        headers: dict = None,
+        auth: tuple = None,
+    ):
+        self.url = self.format_url(url, url_variables or {})
+        self.params = params
+        self.headers = headers
+        self.auth = auth
+        self.start_date = None
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        start_times = [
+            req["start_time"] for req in data_requirements.values() if req["start_time"]
+        ]
+
+        if start_times:
+            oldest_start_time = min(start_times).isoformat()
+            start_time_key = self.params.pop("start_time_key", None)
+            if start_time_key:
+                self.params[start_time_key] = oldest_start_time
+                logging.info(
+                    f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
+                )
+            else:
+                logging.warning("'start_time_key' not found in params.")
+
+        end_times = [
+            req["end_time"] for req in data_requirements.values() if req["end_time"]
+        ]
+
+        if end_times:
+            newest_end_time = max(end_times).isoformat()
+            end_time_key = self.params.pop("end_time_key", None)
+            if end_time_key:
+                self.params[end_time_key] = newest_end_time
+                logging.info(
+                    f"Set end_time to {newest_end_time} and removed 'end_time_key'"
+                )
+            else:
+                logging.warning("'end_time_key' not found in params.")
+
+    def extract(self):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+        response = requests.get(
+            url=self.url,
+            params=self.params,
+            headers=self.headers,
+            auth=self.auth,
+            stream=True,
+        )
+        response.raise_for_status()
+        logging.info(f"Successfully downloaded file from {response.url}")
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data
+
+    @staticmethod
+    def format_url(url_template, url_variables):
+        try:
+            url = url_template.format(**url_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing configuration url_variable: {missing_key}")
+
+        return url

hydroserverpy/etl/extractors/local_file_extractor.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, filepath: str):
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.filepath, "r")
+            logging.info(f"Successfully opened file '{self.filepath}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.filepath}': {e}")
+            return None

hydroserverpy/etl/hydroserver_etl.py

@@ -0,0 +1,40 @@
+import logging
+import pandas as pd
+
+
+class HydroServerETL:
+    def __init__(self, extractor, transformer, loader, source_target_map):
+        self.extractor = extractor
+        self.transformer = transformer
+        self.loader = loader
+        self.source_target_map = source_target_map
+
+    def run(self):
+        """
+        Extracts, transforms, and loads data as defined by the class parameters.
+        """
+
+        # Step 1: Get Target System data requirements from the Loader & prepare parameters for the Extractor
+        data_requirements = self.loader.get_data_requirements(self.source_target_map)
+        self.extractor.prepare_params(data_requirements)
+
+        # Step 2: Extract
+        data = self.extractor.extract()
+        if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+            logging.warning(f"No data was returned from the extractor. Ending ETL run.")
+            return
+        else:
+            logging.info(f"Successfully extracted data.")
+
+        # Step 3: Transform
+        if self.transformer:
+            data = self.transformer.transform(data)
+            if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+                logging.warning(f"No data returned from the transformer. Ending run.")
+                return
+            else:
+                logging.info(f"Successfully transformed data. {data}")
+
+        # Step 4: Load
+        self.loader.load(data, self.source_target_map)
+        logging.info("Successfully loaded data.")

hydroserverpy/etl/loaders/base.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+import pandas as pd
+
+
+class Loader(ABC):
+    @abstractmethod
+    def load(self, *args, **kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def get_data_requirements(self, df: pd.DataFrame) -> Dict[str, pd.Timestamp]:
+        pass

hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -0,0 +1,68 @@
+from hydroserverpy.core.service import HydroServer
+from typing import Dict, Optional
+from .base import Loader
+import logging
+import pandas as pd
+
+
+class HydroServerLoader(HydroServer, Loader):
+    """
+    A class that extends the HydroServer client with ETL-specific functionalities.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+        api_route: str = "api",
+    ):
+        super().__init__(host, username, password, apikey, api_route)
+
+    def load(self, data: pd.DataFrame, source_target_map) -> None:
+        """
+        Load observations from a DataFrame to the HydroServer.
+
+        :param data: A Pandas DataFrame where each column corresponds to a datastream.
+        """
+        data_requirements = self.get_data_requirements(source_target_map)
+        for ds_id in data.columns:
+            if ds_id == "timestamp":
+                continue
+
+            df = data[["timestamp", ds_id]].copy()
+            df.rename(columns={ds_id: "value"}, inplace=True)
+            df.dropna(subset=["value"], inplace=True)
+
+            phenomenon_end_time = data_requirements[ds_id]["start_time"]
+            if phenomenon_end_time:
+                df = df[df["timestamp"] > phenomenon_end_time]
+            if df.empty:
+                logging.warning(
+                    f"No new data to upload for datastream {ds_id}. Skipping."
+                )
+                continue
+            self.datastreams.load_observations(uid=ds_id, observations=df)
+
+    def get_data_requirements(
+        self, source_target_map
+    ) -> Dict[str, Dict[str, pd.Timestamp]]:
+        """
+        Each target system needs to be able to answer the question: 'What data do you need?'
+        and return a time range for each target time series. Usually the answer will be
+        'anything newer than my most recent observation'.
+        """
+        data_requirements = {}
+        for ds_id in source_target_map.values():
+            datastream = self.datastreams.get(uid=ds_id)
+            if not datastream:
+                message = "Couldn't fetch target datastream. ETL process aborted."
+                logging.error(message)
+                raise message
+            start_time = pd.Timestamp(
+                datastream.phenomenon_end_time or "1970-01-01T00:00:00Z"
+            )
+            end_time = pd.Timestamp.now()
+            data_requirements[ds_id] = {"start_time": start_time, "end_time": end_time}
+        return data_requirements

hydroserverpy/etl/transformers/base.py

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+import logging
+import pandas as pd
+
+
+class Transformer(ABC):
+    @abstractmethod
+    def transform(self, *args, **kwargs) -> None:
+        pass
+
+    @property
+    def needs_datastreams(self) -> bool:
+        return False
+
+    @staticmethod
+    def standardize_dataframe(
+        df,
+        datastream_ids,
+        timestamp_column: str = "timestamp",
+        timestamp_format: str = "ISO8601",
+    ):
+        df.rename(
+            columns={timestamp_column: "timestamp", **datastream_ids},
+            inplace=True,
+        )
+
+        # Verify timestamp column is present in the DataFrame
+        if "timestamp" not in df.columns:
+            message = f"Timestamp column '{timestamp_column}' not found in data."
+            logging.error(message)
+            raise ValueError(message)
+
+        # Verify that all datastream_ids are present in the DataFrame
+        expected_columns = set(datastream_ids.values())
+        actual_columns = set(df.columns)
+        missing_datastream_ids = expected_columns - actual_columns
+
+        if missing_datastream_ids:
+            raise ValueError(
+                "The following datastream IDs are specified in the config file but their related keys could not be "
+                f"found in the source system's extracted data: {missing_datastream_ids}"
+            )
+
+        # Keep only 'timestamp' and datastream_id columns
+        columns_to_keep = ["timestamp"] + list(expected_columns)
+        df = df[columns_to_keep]
+
+        # Convert timestamp column to datetime if not already
+        if not pd.api.types.is_datetime64_any_dtype(df["timestamp"]):
+            df["timestamp"] = pd.to_datetime(df["timestamp"], format=timestamp_format)
+
+        return df

hydroserverpy/etl/transformers/csv_transformer.py

@@ -0,0 +1,88 @@
+import logging
+import pandas as pd
+from typing import Dict, Optional, Union
+from .base import Transformer
+
+
+class CSVTransformer(Transformer):
+    def __init__(
+        self,
+        header_row: Optional[int],
+        data_start_row: int,
+        timestamp_column: Union[str, int],
+        datastream_ids: Dict[Union[str, int], str],
+        delimiter: Optional[str] = ",",
+        timestamp_format: Optional[str] = "ISO8601",
+    ):
+        # Pandas is zero-based while CSV is one-based so convert
+        self.header_row = None if header_row is None else header_row - 1
+        self.data_start_row = data_start_row - 1
+        self.timestamp_column = self.convert_to_zero_based(timestamp_column)
+        self.datastream_ids = datastream_ids
+        self.timestamp_format = timestamp_format
+        self.delimiter = delimiter
+
+    def transform(self, data_file) -> Union[pd.DataFrame, None]:
+        """
+        Transforms a CSV file-like object into a Pandas DataFrame where the column
+        names are replaced with their target datastream ids.
+
+        Parameters:
+            data_file: File-like object containing CSV data.
+        Returns:
+            observations_map (dict): Dict mapping datastream IDs to pandas DataFrames.
+        """
+
+        try:
+            df = pd.read_csv(
+                data_file,
+                delimiter=self.delimiter,
+                header=self.header_row,
+                parse_dates=[self.timestamp_column],
+                date_format=self.timestamp_format,
+                skiprows=self.calculate_skiprows(),
+                usecols=[self.timestamp_column] + list(self.datastream_ids.keys()),
+            )
+        except Exception as e:
+            logging.error(f"Error reading CSV data: {e}")
+            return None
+
+        if self.header_row is None:
+            df.columns = list(range(1, len(df.columns) + 1))
+
+        return self.standardize_dataframe(
+            df, self.datastream_ids, self.timestamp_column, self.timestamp_format
+        )
+
+    def calculate_skiprows(self):
+        """
+        Calculates the skiprows parameter for pd.read_csv.
+
+        Returns:
+            skiprows (list or None): List of row indices to skip, or None if no rows need to be skipped.
+        Raises:
+            ValueError: If header_row is not compatible with data_start_row.
+        """
+        if self.data_start_row == 0:
+            if self.header_row is not None:
+                # Cannot have a header row if data starts at the first row
+                raise ValueError(
+                    "header_row must be None when data_start_row is 1 (first row)"
+                )
+            return None  # No rows to skip
+
+        skiprows = list(range(self.data_start_row))
+
+        if self.header_row is not None:
+            if self.header_row >= self.data_start_row:
+                raise ValueError("header_row must be less than data_start_row")
+            if self.header_row in skiprows:
+                # Do not skip the header row
+                skiprows.remove(self.header_row)
+        return skiprows
+
+    @staticmethod
+    def convert_to_zero_based(index: Union[str, int]) -> Union[str, int]:
+        if isinstance(index, int):
+            return index - 1
+        return index

hydroserverpy/etl/transformers/json_transformer.py

@@ -0,0 +1,62 @@
+import logging
+import pandas as pd
+from typing import Dict, Optional, Any, List
+from .base import Transformer
+import json
+import jmespath
+
+
+class JSONTransformer(Transformer):
+    def __init__(
+        self,
+        query_string: str,
+        datastream_ids: Dict[str, str],
+        timestamp_format: Optional[str] = "ISO8601",
+    ):
+        """
+        Initializes the JSONTransformer.
+
+        Parameters:
+            query_string (str): JMESPath to the data array containing time series data.
+            Since JMESPath can natively rename column names, the assumption is the timestamp column
+            is always named 'timestamp' or converted to 'timestamp' in the JMESPath query.
+            datastream_ids (dict): Mapping from JSON field names to datastream IDs.
+            timestamp_format (str, optional): The format of the timestamp, if it needs special parsing.
+        """
+        self.query_string = query_string
+        self.datastream_ids = datastream_ids
+        self.timestamp_format = timestamp_format
+
+    def transform(self, data_file):
+        """
+        Transforms a JSON file-like object into the standard Pandas dataframe format.
+        Since JMESPath can natively rename column names, the assumption is the timestamp column
+        is always named 'timestamp' for JSON data or converted to 'timestamp' in the JMESPath query.
+
+        Parameters:
+            data_file: File-like object containing JSON data.
+
+        Returns:
+            pd.DataFrame: pandas DataFrames in the format pd.Timestamp, datastream_id_1, datastream_id_2, ...
+        """
+        json_data = json.load(data_file)
+        data_points = self.extract_data_points(json_data)
+        if not data_points:
+            logging.warning("No data points found in the JSON data.")
+            return None
+
+        df = pd.DataFrame(data_points)
+
+        return self.standardize_dataframe(
+            df,
+            self.datastream_ids,
+            timestamp_format=self.timestamp_format,
+        )
+
+    def extract_data_points(self, json_data: Any) -> Optional[List[dict]]:
+        """Extracts data points from the JSON data using the data_path."""
+        data_points = jmespath.search(self.query_string, json_data)
+
+        if isinstance(data_points, dict):
+            data_points = [data_points]
+        return data_points

hydroserverpy/etl/types.py

@@ -0,0 +1,7 @@
+from typing import TypedDict
+import pandas as pd
+
+
+class TimeRange(TypedDict):
+    start_time: pd.Timestamp
+    end_time: pd.Timestamp