hydroserverpy 1.3.1__py3-none-any.whl → 1.4.0b4__py3-none-any.whl

hydroserverpy/api/models/etl/loaders/hydroserver_loader.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from .base import Loader
+import logging
+import pandas as pd
+from ..etl_configuration import Payload, SourceTargetMapping
+
+if TYPE_CHECKING:
+    from hydroserverpy.api.client import HydroServer
+
+
+class HydroServerLoader(Loader):
+    """
+    A class that extends the HydroServer client with ETL-specific functionalities.
+    """
+
+    def __init__(self, client: HydroServer, data_source_id):
+        self.client = client
+        self._begin_cache: dict[str, pd.Timestamp] = {}
+        self.data_source_id = data_source_id
+
+    def load(self, data: pd.DataFrame, payload: Payload) -> None:
+        """
+        Load observations from a DataFrame to the HydroServer.
+        :param data: A Pandas DataFrame where each column corresponds to a datastream.
+        """
+        begin_date = self.earliest_begin_date(payload)
+        new_data = data[data["timestamp"] > begin_date]
+        for col in new_data.columns.difference(["timestamp"]):
+            df = (
+                new_data[["timestamp", col]]
+                .rename(columns={col: "value"})
+                .dropna(subset=["value"])
+            )
+            if df.empty:
+                logging.warning(f"No new data for {col}, skipping.")
+                continue
+            logging.info(f"loading dataframe {df}")
+            logging.info(f"dtypes: {df.dtypes}")
+
+            df = df.rename(columns={"timestamp": "phenomenon_time", "value": "result"})
+
+            # Chunked upload
+            CHUNK_SIZE = 5000
+            total = len(df)
+            for start in range(0, total, CHUNK_SIZE):
+                end = min(start + CHUNK_SIZE, total)
+                chunk = df.iloc[start:end]
+                logging.info(
+                    "Uploading %s rows (%s-%s) to datastream %s",
+                    len(chunk),
+                    start,
+                    end - 1,
+                    col,
+                )
+                try:
+                    self.client.datastreams.load_observations(
+                        uid=str(col), observations=chunk
+                    )
+                except Exception as e:
+                    status = getattr(e, "status_code", None) or getattr(
+                        getattr(e, "response", None), "status_code", None
+                    )
+                    if status == 409 or "409" in str(e) or "Conflict" in str(e):
+                        logging.info(
+                            "409 Conflict for datastream %s on rows %s-%s; skipping remainder for this stream.",
+                            col,
+                            start,
+                            end - 1,
+                        )
+                        break
+                    raise
+
+    def _fetch_earliest_begin(
+        self, mappings: list[SourceTargetMapping]
+    ) -> pd.Timestamp:
+        logging.info("Querying HydroServer for earliest begin date for payload...")
+        timestamps = []
+        datastreams = self.client.datastreams.list(
+            data_source=self.data_source_id
+        ).items
+        ds_by_uid = {str(ds.uid): ds for ds in datastreams}
+        for m in mappings:
+            for p in m.paths:
+                datastream = ds_by_uid[str(p.target_identifier)]
+                raw = datastream.phenomenon_end_time or "1970-01-01"
+                ts = pd.to_datetime(raw, utc=True)
+                timestamps.append(ts)
+        logging.info(f"Found earliest begin date: {min(timestamps)}")
+        return min(timestamps)
+
+    def earliest_begin_date(self, payload: Payload) -> pd.Timestamp:
+        """
+        Return earliest begin date for a payload, or compute+cache it on first call.
+        """
+        key = payload.name
+        if key not in self._begin_cache:
+            self._begin_cache[key] = self._fetch_earliest_begin(payload.mappings)
+        return self._begin_cache[key]

hydroserverpy/api/models/etl/schedule.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from typing import Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class Schedule(BaseModel):
+    interval: int = Field(..., gt=0)
+    interval_units: Optional[Literal["minutes", "hours", "days"]] = Field(
+        None, alias="intervalUnits"
+    )
+    crontab: Optional[str]
+    start_time: Optional[datetime] = Field(None, alias="startTime")
+    end_time: Optional[datetime] = Field(None, alias="endTime")
+
+    class Config:
+        allow_population_by_field_name = True

hydroserverpy/api/models/etl/status.py

@@ -0,0 +1,14 @@
+from datetime import datetime
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class Status(BaseModel):
+    paused: bool = Field(False)
+    last_run_successful: Optional[bool] = Field(None, alias="lastRunSuccessful")
+    last_run_message: Optional[str] = Field(None, alias="lastRunMessage")
+    last_run: Optional[datetime] = Field(None, alias="lastRun")
+    next_run: Optional[datetime] = Field(None, alias="nextRun")
+
+    class Config:
+        allow_population_by_field_name = True

hydroserverpy/{etl → api/models/etl}/timestamp_parser.py

@@ -73,7 +73,10 @@ class TimestampParser:
         return localized.dt.tz_convert(timezone.utc)
 
     def parse_series(self, raw_series: pd.Series) -> pd.Series:
-        s = raw_series.str.strip()
+        if pd.api.types.is_datetime64_any_dtype(raw_series):
+            s = raw_series  # already datetimes
+        else:
+            s = raw_series.astype("string", copy=False).str.strip()
         parsed = self._convert_series_to_UTC(s)
 
         if parsed.isna().any():

hydroserverpy/api/models/etl/transformers/__init__.py

@@ -0,0 +1,5 @@
+from .base import Transformer
+from .json_transformer import JSONTransformer
+from .csv_transformer import CSVTransformer
+
+__all__ = ["Transformer", "JSONTransformer", "CSVTransformer"]

hydroserverpy/api/models/etl/transformers/base.py

@@ -0,0 +1,137 @@
+from abc import ABC, abstractmethod
+import ast
+from functools import lru_cache
+import logging
+import re
+from typing import List, Union
+import pandas as pd
+
+from ..timestamp_parser import TimestampParser
+from ..etl_configuration import MappingPath, TransformerConfig, SourceTargetMapping
+
+ALLOWED_AST = (
+    ast.Expression,
+    ast.BinOp,
+    ast.UnaryOp,
+    ast.Add,
+    ast.Sub,
+    ast.Mult,
+    ast.Div,
+    ast.UAdd,
+    ast.USub,
+    ast.Name,
+    ast.Load,
+    ast.Constant,
+)
+
+
+def _canonicalize_expr(expr: str) -> str:
+    # normalize whitespace for cache hits; parentheses remain intact
+    return re.sub(r"\s+", "", expr)
+
+
+@lru_cache(maxsize=256)
+def _compile_arithmetic_expr_canon(expr_no_ws: str):
+    tree = ast.parse(expr_no_ws, mode="eval")
+    for node in ast.walk(tree):
+        if not isinstance(node, ALLOWED_AST):
+            raise ValueError(
+                "Only +, -, *, / with 'x' and numeric literals are allowed."
+            )
+        if isinstance(node, ast.Name) and node.id != "x":
+            raise ValueError("Only the variable 'x' is allowed.")
+        if isinstance(node, ast.Constant):
+            val = node.value
+            if isinstance(val, bool) or not isinstance(val, (int, float)):
+                raise ValueError("Only numeric literals are allowed.")
+    return compile(tree, "<expr>", "eval")
+
+
+def _compile_arithmetic_expr(expr: str):
+    return _compile_arithmetic_expr_canon(_canonicalize_expr(expr))
+
+
+class Transformer(ABC):
+    def __init__(self, transformer_config: TransformerConfig):
+        self.cfg = transformer_config
+        self.timestamp = transformer_config.timestamp
+        self.timestamp_parser = TimestampParser(self.timestamp)
+
+    @abstractmethod
+    def transform(self, *args, **kwargs) -> None:
+        pass
+
+    @property
+    def needs_datastreams(self) -> bool:
+        return False
+
+    def standardize_dataframe(
+        self, df: pd.DataFrame, mappings: List[SourceTargetMapping]
+    ):
+        logging.info(f"Successfully read payload into dataframe:\n {df}")
+
+        # 1) Normalize timestamp column
+        df.rename(columns={self.timestamp.key: "timestamp"}, inplace=True)
+        if "timestamp" not in df.columns:
+            msg = f"Timestamp column '{self.timestamp.key}' not found in data."
+            logging.error(msg)
+            raise ValueError(msg)
+        logging.info(f"Renamed timestamp column to 'timestamp'")
+
+        df["timestamp"] = self.timestamp_parser.parse_series(df["timestamp"])
+        logging.info(f"Normalized timestamp column \n {df}")
+
+        df = df.drop_duplicates(subset=["timestamp"], keep="last")
+        logging.info(f"Removed duplicates\n")
+
+        def _resolve_source_col(s_id: Union[str, int]) -> str:
+            if isinstance(s_id, int) and s_id not in df.columns:
+                try:
+                    return df.columns[s_id]
+                except IndexError:
+                    raise ValueError(
+                        f"Source index {s_id} is out of range for extracted data."
+                    )
+            if s_id not in df.columns:
+                raise ValueError(f"Source column '{s_id}' not found in extracted data.")
+            return s_id
+
+        def _apply_transformations(series: pd.Series, path: MappingPath) -> pd.Series:
+            out = series  # accumulator for sequential transforms
+            if out.dtype == "object":
+                out = pd.to_numeric(out, errors="coerce")
+
+            for transformation in path.data_transformations:
+                if transformation.type == "expression":
+                    code = _compile_arithmetic_expr(transformation.expression)
+                    try:
+                        out = eval(code, {"__builtins__": {}}, {"x": out})
+                    except Exception as ee:
+                        logging.exception(
+                            "Data transformation failed for expression=%r",
+                            transformation.expression,
+                        )
+                        raise
+                else:
+                    msg = f"Unsupported transformation type: {transformation.type}"
+                    logging.error(msg)
+                    raise ValueError(msg)
+            return out
+
+        # source target mappings may be one to many. Therefore, create a new column for each target and apply transformations
+        transformed_df = pd.DataFrame(index=df.index)
+        for m in mappings:
+            src_col = _resolve_source_col(m.source_identifier)
+            base = df[src_col]
+            for path in m.paths:
+                target_col = str(path.target_identifier)
+                transformed_df[target_col] = _apply_transformations(base, path)
+
+        logging.info(f"Mapped payload sources to targets")
+
+        # 6) Keep only timestamp + target columns
+        df = pd.concat([df[["timestamp"]], pd.DataFrame(transformed_df)], axis=1)
+
+        logging.info(f"standardized dataframe created: {df.shape}")
+
+        return df

hydroserverpy/{etl → api/models/etl}/transformers/csv_transformer.py

@@ -1,25 +1,28 @@
 from io import StringIO
 import logging
 import pandas as pd
-from typing import Iterable, Union
+from typing import Iterable, List, Union
 from .base import Transformer
+from ..etl_configuration import TransformerConfig, SourceTargetMapping
 
 
 class CSVTransformer(Transformer):
-    def __init__(self, settings: object):
-        super().__init__(settings)
+    def __init__(self, transformer_config: TransformerConfig):
+        super().__init__(transformer_config)
 
         # Pandas is zero-based while CSV is one-based so convert
         self.header_row = (
-            None if settings.get("headerRow") is None else settings["headerRow"] - 1
+            None if self.cfg.header_row is None else self.cfg.header_row - 1
         )
         self.data_start_row = (
-            settings["dataStartRow"] - 1 if "dataStartRow" in settings else 0
+            self.cfg.data_start_row - 1 if self.cfg.data_start_row else 0
         )
-        self.delimiter = settings.get("delimiter", ",")
-        self.identifier_type = settings.get("identifierType", "name")
+        self.delimiter = self.cfg.delimiter or ","
+        self.identifier_type = self.cfg.identifier_type or "name"
 
-    def transform(self, data_file, mappings) -> Union[pd.DataFrame, None]:
+    def transform(
+        self, data_file, mappings: List[SourceTargetMapping]
+    ) -> Union[pd.DataFrame, None]:
         """
         Transforms a CSV file-like object into a Pandas DataFrame where the column
         names are replaced with their target datastream ids.
@@ -31,7 +34,14 @@ class CSVTransformer(Transformer):
         """
 
         clean_file = self._strip_comments(data_file)
-        source_identifiers = [mapping["sourceIdentifier"] for mapping in mappings]
+        use_index = self.identifier_type == "index"
+
+        if use_index:
+            # Users will always interact in 1-based, so if the key is a column index, convert to 0-based to work with Pandas
+            timestamp_pos = int(self.timestamp.key) - 1
+            usecols = [timestamp_pos] + [int(m.source_identifier) - 1 for m in mappings]
+        else:
+            usecols = [self.timestamp.key] + [m.source_identifier for m in mappings]
 
         try:
             # Pandas’ heuristics strip offsets and silently coerce failures to strings.
@@ -42,16 +52,17 @@ class CSVTransformer(Transformer):
                 sep=self.delimiter,
                 header=self.header_row,
                 skiprows=self._build_skiprows(),
-                usecols=[self.timestamp_key] + source_identifiers,
-                dtype={self.timestamp_key: "string"},
+                usecols=usecols,
+                dtype={self.timestamp.key: "string"},
             )
             logging.info(f"CSV file read into dataframe: {df.shape}")
         except Exception as e:
             logging.error(f"Error reading CSV data: {e}")
             return None
 
-        if self.header_row is None:
-            df.columns = list(range(1, len(df.columns) + 1))
+        # In index mode, relabel columns back to original 1-based indices so base transformer can use integer labels directly
+        if use_index:
+            df.columns = [(c + 1) if isinstance(c, int) else c for c in usecols]
 
         return self.standardize_dataframe(df, mappings)
 

hydroserverpy/{etl → api/models/etl}/transformers/json_transformer.py

@@ -1,17 +1,18 @@
 import logging
 import pandas as pd
-from typing import Dict, Optional, Any, List
+from typing import Optional, Any, List
 from .base import Transformer
 import json
 import jmespath
+from ..etl_configuration import TransformerConfig, SourceTargetMapping
 
 
 class JSONTransformer(Transformer):
-    def __init__(self, settings: object):
-        super().__init__(settings)
-        self.JMESPath = settings["JMESPath"]
+    def __init__(self, transformer_config: TransformerConfig):
+        super().__init__(transformer_config)
+        self.jmespath = transformer_config.jmespath
 
-    def transform(self, data_file, mappings):
+    def transform(self, data_file, mappings: List[SourceTargetMapping]):
         """
         Transforms a JSON file-like object into the standard Pandas dataframe format.
         Since JMESPath can natively rename column names, the assumption is the timestamp column
@@ -23,7 +24,21 @@ class JSONTransformer(Transformer):
         Returns:
             pd.DataFrame: pandas DataFrames in the format pd.Timestamp, datastream_id_1, datastream_id_2, ...
         """
+        if data_file is None:
+            raise TypeError(
+                "JSONTransformer received None; expected file-like, bytes, or str"
+            )
+
         json_data = json.load(data_file)
+        logging.info(f"Read in json data: \n{data_file}")
+        logging.info(
+            "JSONTransformer cfg:\n jmespath=%r\n ts.key=%r\n ts.format=%r\n ts.custom=%r",
+            self.cfg.jmespath,
+            self.timestamp.key,
+            self.timestamp.format,
+            self.timestamp.custom_format,
+        )
+
         data_points = self.extract_data_points(json_data)
         if not data_points:
             logging.warning("No data points found in the JSON data.")
@@ -35,7 +50,7 @@ class JSONTransformer(Transformer):
 
     def extract_data_points(self, json_data: Any) -> Optional[List[dict]]:
         """Extracts data points from the JSON data using the data_path."""
-        data_points = jmespath.search(self.JMESPath, json_data)
+        data_points = jmespath.search(self.jmespath, json_data)
 
         if isinstance(data_points, dict):
             data_points = [data_points]

hydroserverpy/api/services/etl/data_source.py

@@ -77,10 +77,7 @@ class DataSourceService(HydroServerBaseService):
                 "paused": paused,
             },
             "datastreamIds": (
-                [
-                    normalize_uuid(datastream)
-                    for datastream in datastreams
-                ]
+                [normalize_uuid(datastream) for datastream in datastreams]
                 if datastreams
                 else []
             ),

{hydroserverpy-1.3.1.dist-info → hydroserverpy-1.4.0b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.3.1
+Version: 1.4.0b4
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.3.1.dist-info → hydroserverpy-1.4.0b4.dist-info}/RECORD

@@ -1,14 +1,32 @@
-hydroserverpy/__init__.py,sha256=gn3x_C6Pe1Dn90uXn7yIwEhaQm5DE76MhamdMOqF2yM,220
+hydroserverpy/__init__.py,sha256=xnuWIehUYshy05GptyIvHiD52FOjoWOloXAfT1LgP3U,150
 hydroserverpy/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/client.py,sha256=jduKZV2cOkPVRjIjAiVYnTncMfEtW6IaCb895Y_PfiI,5697
 hydroserverpy/api/utils.py,sha256=1RUglpvegBZOcu9BEExxsAzaGOyu4tdUk2JyiBEbzxI,496
 hydroserverpy/api/models/__init__.py,sha256=NLq95t1oC2co5aqVYSw9Pq0RAsLHnLjNq1tsgbMepTg,773
 hydroserverpy/api/models/base.py,sha256=mQZbanDg9t2GN9mOR_XOOtAfYF7AkY0fBZ6fHat6NRs,6944
-hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydroserverpy/api/models/etl/__init__.py,sha256=_D8_Nbs06-47wwsHOAF9tOohQYQ52gOhz_NUyfloMUw,699
 hydroserverpy/api/models/etl/data_archive.py,sha256=rnmD_FQ1yjJ0KPBigylAQ3uQ6QBppJtBopJK4oCPLSo,2613
-hydroserverpy/api/models/etl/data_source.py,sha256=4s5JfpF00Heir9T1oc_KAUdI3z5Jj8ce8R56KqQqm5A,3959
+hydroserverpy/api/models/etl/data_source.py,sha256=YWSdudFONe1eniGBkruLRAP8BDyt0tGmZD8hzzByjKU,5123
+hydroserverpy/api/models/etl/etl_configuration.py,sha256=anD_0zlldJKogie15j9SIabJvNqnY5fz5xjuFsFUFwU,6034
+hydroserverpy/api/models/etl/factories.py,sha256=-inTw_C694YieDU4vbrm1qyeZMEYZqVhHSyEQJLMueo,802
 hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJFJg6aGogW5ZZk7fA17y4p--yWwiOhZ0,1367
 hydroserverpy/api/models/etl/orchestration_system.py,sha256=5wdGsXCMqHfE3--zG-3WAPAVPNMPIx99y-7UUhdCink,2060
+hydroserverpy/api/models/etl/schedule.py,sha256=-TxRpYSFbyYkzAPBWOh5udx1s6v1SvLl3_LE2j_b1uE,512
+hydroserverpy/api/models/etl/status.py,sha256=vYT7go7DMcOgy29w0yhHpKz6AdprLmOxWZE9G_DHVdw,503
+hydroserverpy/api/models/etl/timestamp_parser.py,sha256=lDnParK2j2M9TF7qspJDeKFGGpO4d1F2KJEKZ4xH5Yw,4374
+hydroserverpy/api/models/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
+hydroserverpy/api/models/etl/extractors/__init__.py,sha256=Z0viw2vk96Ytpz3n7ODtkYz9Zx0I0NsZUbna2ZWvhkw,243
+hydroserverpy/api/models/etl/extractors/base.py,sha256=uLAdi1PrOVMtuCU1ZN_liBW_ElD2mklrBrQ_AZZQtNw,1949
+hydroserverpy/api/models/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
+hydroserverpy/api/models/etl/extractors/http_extractor.py,sha256=AgS0vDmHhN3do1FII-hNEvkK40lDjlS1iftHplWd1No,805
+hydroserverpy/api/models/etl/extractors/local_file_extractor.py,sha256=AwC0T-F8D0S7zR0MUIQXKLfv9b0uU60YoUW615lgNl0,648
+hydroserverpy/api/models/etl/loaders/__init__.py,sha256=rEqYo1Tim7Fzrp1jPhV_yn3ll90dUGMAjcieEqh_4Pk,118
+hydroserverpy/api/models/etl/loaders/base.py,sha256=J3dqm_b6BmEsF7VR3sUxBVQpLJsRx7fTIir5v5TORE0,229
+hydroserverpy/api/models/etl/loaders/hydroserver_loader.py,sha256=0qdsSu4bLD4R86eSyQY2Qwg1HmMsruVjFhBKCrErsBU,3868
+hydroserverpy/api/models/etl/transformers/__init__.py,sha256=YQhjdoRdq4xikLWETnsRIaMvCae5flRpPrfw9lj9pOA,184
+hydroserverpy/api/models/etl/transformers/base.py,sha256=Qt9U6rRFS8aq142n0Cig1wMkVC9-4IBiXIE1nAjEPDY,4971
+hydroserverpy/api/models/etl/transformers/csv_transformer.py,sha256=06AWV9S9we4LRQLpn5WMVl7sX6ylDKPb2KHNC0Jiu7o,3478
+hydroserverpy/api/models/etl/transformers/json_transformer.py,sha256=puKQI8abWJEQTcw34lEHgHjuPOuzcSBv95_txErzchk,2116
 hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
 hydroserverpy/api/models/iam/apikey.py,sha256=Z4iXg_K056naT3ogwc5wzyNnRpxHkOCz0lk-Gim4eL8,3146
@@ -28,7 +46,7 @@ hydroserverpy/api/services/__init__.py,sha256=Nb7rc1Zt8kpRElgFdWPdcyUDrtm7XdJDgz
 hydroserverpy/api/services/base.py,sha256=f7CoQ1m-pdgVwqJsdvE7vcannw-3i7yJgBMI4eHZxAQ,3725
 hydroserverpy/api/services/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/services/etl/data_archive.py,sha256=-Pmv9EqNJncVX3gPDIeNM4TsR6fHgOIjmMGt9fGOeYg,5842
-hydroserverpy/api/services/etl/data_source.py,sha256=xR_GQA7IRi-2bMrF6m_kWSmx2xi3knH-GWbGTA3LoQs,5831
+hydroserverpy/api/services/etl/data_source.py,sha256=XWWgbVyhyZxRt4s6wBc9-lnv_O86Bte1Vk3_Aza4HGY,5773
 hydroserverpy/api/services/etl/orchestration_system.py,sha256=Otj_DiFpFBQzSc4Ei7LxneBf3VPnodI0pqoQM2BldcM,1935
 hydroserverpy/api/services/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/services/iam/role.py,sha256=PV0odC_lL9kV3ggrTjAUEMTo0WNUzv4AeMHNAXlkbN8,1137
@@ -41,30 +59,11 @@ hydroserverpy/api/services/sta/result_qualifier.py,sha256=gkgofUqzGXgdkyAvK9RW_d
 hydroserverpy/api/services/sta/sensor.py,sha256=SmrIFNHD_vrlnbZvzsv0Wf0Pexk2oDWQ28LtWdj2kao,3274
 hydroserverpy/api/services/sta/thing.py,sha256=Hyo3zTghSs7IIdsOGRu35i9w-aGOYlK9bl2AnmU4bBs,6666
 hydroserverpy/api/services/sta/unit.py,sha256=NFToSAIGTwDfwYWe8Q-I_f5xsw_GYzFEkMnhSJ-ChvE,2178
-hydroserverpy/etl/__init__.py,sha256=qK2m4LZl8czR3VE8SxrlipSy5tLGLNB60lxD7dD0GjU,659
-hydroserverpy/etl/hydroserver_etl.py,sha256=FSdvM3T7QHEWWulWRT8t-FMHSxAGB4GvleUXtSk5IWc,1507
-hydroserverpy/etl/timestamp_parser.py,sha256=MA_a0qPExbIQGt-ju7w6WflVDMzigW1LKUFCJ_jhkp4,4218
-hydroserverpy/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
-hydroserverpy/etl/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/etl/extractors/base.py,sha256=mK8WotEcG-4cHIW3ExS03wxyKtXPzcDhmo8S_5CGnek,1989
-hydroserverpy/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
-hydroserverpy/etl/extractors/http_extractor.py,sha256=WxWyg-GLyr6Rb-2uCFniWe6Nmk71x-frmxgEYTr9juU,814
-hydroserverpy/etl/extractors/local_file_extractor.py,sha256=WZ4xIg5FiJ5GbVuR71Uj9tw_vVyzGYeweWctKscUSW0,563
-hydroserverpy/etl/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/etl/loaders/base.py,sha256=q3pTp8NqZUYF1IxwKp7TOA5b4HuJkhz3FD9tIqpL7iM,273
-hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=N4zu_PefOwMr-NoFvq0g57VumYpNtD6o76oqhmF35ts,2545
-hydroserverpy/etl/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/etl/transformers/base.py,sha256=BtRNQItt6VY9r1TBMHByOTzOB1rY1QdY8ijqCgl0riI,2259
-hydroserverpy/etl/transformers/csv_transformer.py,sha256=0kWfRKPwiGxCNZ87Q4SiBlfM3PuKL6upc1ljphBY89o,2891
-hydroserverpy/etl/transformers/json_transformer.py,sha256=R7tSyDB4Wn1snP75ctbEDMaMCdjyhPnMzN_W2VV3Mv4,1506
-hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
-hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=zZDIpbBTUdm4-9G3gJ8F_IqsLvP5wtGvr4Xy6_5K3tQ,14181
 hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
 hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
-hydroserverpy-1.3.1.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
-hydroserverpy-1.3.1.dist-info/METADATA,sha256=7p10qFkxjJqbUJglmqEF300RygoDzFsvpHFU0f9STEw,530
-hydroserverpy-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hydroserverpy-1.3.1.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
-hydroserverpy-1.3.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-hydroserverpy-1.3.1.dist-info/RECORD,,
+hydroserverpy-1.4.0b4.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
+hydroserverpy-1.4.0b4.dist-info/METADATA,sha256=GCgulq3Im1uhFlRJtRg54dwOIqHK3wFfxAXpT6hlDlA,532
+hydroserverpy-1.4.0b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hydroserverpy-1.4.0b4.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
+hydroserverpy-1.4.0b4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+hydroserverpy-1.4.0b4.dist-info/RECORD,,

hydroserverpy/etl/__init__.py

@@ -1,21 +0,0 @@
-from .extractors.local_file_extractor import LocalFileExtractor
-from .extractors.ftp_extractor import FTPExtractor
-from .extractors.http_extractor import HTTPExtractor
-from .transformers.csv_transformer import CSVTransformer
-from .transformers.json_transformer import JSONTransformer
-from .transformers.base import Transformer
-from .extractors.base import Extractor
-from .loaders.base import Loader
-from .loaders.hydroserver_loader import HydroServerLoader
-
-__all__ = [
-    "CSVTransformer",
-    "JSONTransformer",
-    "LocalFileExtractor",
-    "FTPExtractor",
-    "HTTPExtractor",
-    "Extractor",
-    "Transformer",
-    "Loader",
-    "HydroServerLoader",
-]

hydroserverpy/etl/extractors/local_file_extractor.py

@@ -1,19 +0,0 @@
-import logging
-from .base import Extractor
-
-
-class LocalFileExtractor(Extractor):
-    def __init__(self, settings: object):
-        super().__init__(settings)
-
-    def extract(self):
-        """
-        Opens the file and returns a file-like object.
-        """
-        try:
-            file_handle = open(self.source_uri, "r")
-            logging.info(f"Successfully opened file '{self.source_uri}'.")
-            return file_handle
-        except Exception as e:
-            logging.error(f"Error opening file '{self.source_uri}': {e}")
-            return None

hydroserverpy/etl/hydroserver_etl.py

@@ -1,40 +0,0 @@
-import logging
-import pandas as pd
-
-
-class HydroServerETL:
-    def __init__(self, extractor, transformer, loader, source_target_map):
-        self.extractor = extractor
-        self.transformer = transformer
-        self.loader = loader
-        self.source_target_map = source_target_map
-
-    def run(self):
-        """
-        Extracts, transforms, and loads data as defined by the class parameters.
-        """
-
-        # Step 1: Get Target System data requirements from the Loader & prepare parameters for the Extractor
-        data_requirements = self.loader.get_data_requirements(self.source_target_map)
-        self.extractor.prepare_params(data_requirements)
-
-        # Step 2: Extract
-        data = self.extractor.extract()
-        if data is None or (isinstance(data, pd.DataFrame) and data.empty):
-            logging.warning(f"No data was returned from the extractor. Ending ETL run.")
-            return
-        else:
-            logging.info(f"Successfully extracted data.")
-
-        # Step 3: Transform
-        if self.transformer:
-            data = self.transformer.transform(data)
-            if data is None or (isinstance(data, pd.DataFrame) and data.empty):
-                logging.warning(f"No data returned from the transformer. Ending run.")
-                return
-            else:
-                logging.info(f"Successfully transformed data. {data}")
-
-        # Step 4: Load
-        self.loader.load(data, self.source_target_map)
-        logging.info("Successfully loaded data.")