hydroserverpy 1.4.0b3__py3-none-any.whl → 1.4.0b4__py3-none-any.whl

hydroserverpy/api/models/etl/extractors/http_extractor.py

@@ -17,11 +17,8 @@ class HTTPExtractor(Extractor):
         url = self.resolve_placeholder_variables(payload, loader)
         logging.info(f"Requesting data from → {url}")
 
-        try:
-            response = requests.get(url)
-        except Exception as e:
-            logging.error(f"Failed to fetch {url}: {e}")
-            raise
+        response = requests.get(url)
+        response.raise_for_status()
 
         data = BytesIO()
         for chunk in response.iter_content(chunk_size=8192):

hydroserverpy/api/models/etl/schedule.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from typing import Literal, Optional
-from pydantic import BaseModel, Field, EmailStr
+from pydantic import BaseModel, Field
 
 
 class Schedule(BaseModel):

hydroserverpy/api/models/etl/timestamp_parser.py

@@ -73,7 +73,10 @@ class TimestampParser:
         return localized.dt.tz_convert(timezone.utc)
 
     def parse_series(self, raw_series: pd.Series) -> pd.Series:
-        s = raw_series.str.strip()
+        if pd.api.types.is_datetime64_any_dtype(raw_series):
+            s = raw_series  # already datetimes
+        else:
+            s = raw_series.astype("string", copy=False).str.strip()
         parsed = self._convert_series_to_UTC(s)
 
         if parsed.isna().any():

hydroserverpy/api/models/etl/transformers/base.py

@@ -68,14 +68,21 @@ class Transformer(ABC):
     def standardize_dataframe(
         self, df: pd.DataFrame, mappings: List[SourceTargetMapping]
     ):
+        logging.info(f"Successfully read payload into dataframe:\n {df}")
+
         # 1) Normalize timestamp column
         df.rename(columns={self.timestamp.key: "timestamp"}, inplace=True)
         if "timestamp" not in df.columns:
             msg = f"Timestamp column '{self.timestamp.key}' not found in data."
             logging.error(msg)
             raise ValueError(msg)
+        logging.info(f"Renamed timestamp column to 'timestamp'")
+
         df["timestamp"] = self.timestamp_parser.parse_series(df["timestamp"])
+        logging.info(f"Normalized timestamp column \n {df}")
+
         df = df.drop_duplicates(subset=["timestamp"], keep="last")
+        logging.info(f"Removed duplicates\n")
 
         def _resolve_source_col(s_id: Union[str, int]) -> str:
             if isinstance(s_id, int) and s_id not in df.columns:
@@ -120,6 +127,8 @@ class Transformer(ABC):
                 target_col = str(path.target_identifier)
                 transformed_df[target_col] = _apply_transformations(base, path)
 
+        logging.info(f"Mapped payload sources to targets")
+
         # 6) Keep only timestamp + target columns
         df = pd.concat([df[["timestamp"]], pd.DataFrame(transformed_df)], axis=1)
 

hydroserverpy/api/models/etl/transformers/json_transformer.py

@@ -24,7 +24,21 @@ class JSONTransformer(Transformer):
         Returns:
             pd.DataFrame: pandas DataFrames in the format pd.Timestamp, datastream_id_1, datastream_id_2, ...
         """
+        if data_file is None:
+            raise TypeError(
+                "JSONTransformer received None; expected file-like, bytes, or str"
+            )
+
         json_data = json.load(data_file)
+        logging.info(f"Read in json data: \n{data_file}")
+        logging.info(
+            "JSONTransformer cfg:\n jmespath=%r\n ts.key=%r\n ts.format=%r\n ts.custom=%r",
+            self.cfg.jmespath,
+            self.timestamp.key,
+            self.timestamp.format,
+            self.timestamp.custom_format,
+        )
+
         data_points = self.extract_data_points(json_data)
         if not data_points:
             logging.warning("No data points found in the JSON data.")

{hydroserverpy-1.4.0b3.dist-info → hydroserverpy-1.4.0b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.4.0b3
+Version: 1.4.0b4
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.4.0b3.dist-info → hydroserverpy-1.4.0b4.dist-info}/RECORD

@@ -11,22 +11,22 @@ hydroserverpy/api/models/etl/etl_configuration.py,sha256=anD_0zlldJKogie15j9SIab
 hydroserverpy/api/models/etl/factories.py,sha256=-inTw_C694YieDU4vbrm1qyeZMEYZqVhHSyEQJLMueo,802
 hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJFJg6aGogW5ZZk7fA17y4p--yWwiOhZ0,1367
 hydroserverpy/api/models/etl/orchestration_system.py,sha256=5wdGsXCMqHfE3--zG-3WAPAVPNMPIx99y-7UUhdCink,2060
-hydroserverpy/api/models/etl/schedule.py,sha256=b3ffBPFn5jVnEJnG8YlC_93_OKcmC_kpgwRpn1dNmCo,522
+hydroserverpy/api/models/etl/schedule.py,sha256=-TxRpYSFbyYkzAPBWOh5udx1s6v1SvLl3_LE2j_b1uE,512
 hydroserverpy/api/models/etl/status.py,sha256=vYT7go7DMcOgy29w0yhHpKz6AdprLmOxWZE9G_DHVdw,503
-hydroserverpy/api/models/etl/timestamp_parser.py,sha256=MA_a0qPExbIQGt-ju7w6WflVDMzigW1LKUFCJ_jhkp4,4218
+hydroserverpy/api/models/etl/timestamp_parser.py,sha256=lDnParK2j2M9TF7qspJDeKFGGpO4d1F2KJEKZ4xH5Yw,4374
 hydroserverpy/api/models/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
 hydroserverpy/api/models/etl/extractors/__init__.py,sha256=Z0viw2vk96Ytpz3n7ODtkYz9Zx0I0NsZUbna2ZWvhkw,243
 hydroserverpy/api/models/etl/extractors/base.py,sha256=uLAdi1PrOVMtuCU1ZN_liBW_ElD2mklrBrQ_AZZQtNw,1949
 hydroserverpy/api/models/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
-hydroserverpy/api/models/etl/extractors/http_extractor.py,sha256=sZl9PVTU9QbAreEu5zrIFeMLyIUkkiUAT8AOrbde5gc,892
+hydroserverpy/api/models/etl/extractors/http_extractor.py,sha256=AgS0vDmHhN3do1FII-hNEvkK40lDjlS1iftHplWd1No,805
 hydroserverpy/api/models/etl/extractors/local_file_extractor.py,sha256=AwC0T-F8D0S7zR0MUIQXKLfv9b0uU60YoUW615lgNl0,648
 hydroserverpy/api/models/etl/loaders/__init__.py,sha256=rEqYo1Tim7Fzrp1jPhV_yn3ll90dUGMAjcieEqh_4Pk,118
 hydroserverpy/api/models/etl/loaders/base.py,sha256=J3dqm_b6BmEsF7VR3sUxBVQpLJsRx7fTIir5v5TORE0,229
 hydroserverpy/api/models/etl/loaders/hydroserver_loader.py,sha256=0qdsSu4bLD4R86eSyQY2Qwg1HmMsruVjFhBKCrErsBU,3868
 hydroserverpy/api/models/etl/transformers/__init__.py,sha256=YQhjdoRdq4xikLWETnsRIaMvCae5flRpPrfw9lj9pOA,184
-hydroserverpy/api/models/etl/transformers/base.py,sha256=iFfoxsaq4IRiFZgcC2ooRikyGmPexTHIllvf4shf9F4,4662
+hydroserverpy/api/models/etl/transformers/base.py,sha256=Qt9U6rRFS8aq142n0Cig1wMkVC9-4IBiXIE1nAjEPDY,4971
 hydroserverpy/api/models/etl/transformers/csv_transformer.py,sha256=06AWV9S9we4LRQLpn5WMVl7sX6ylDKPb2KHNC0Jiu7o,3478
-hydroserverpy/api/models/etl/transformers/json_transformer.py,sha256=em3tIoAmEnOFPfHzGmUTTGNMPeEoWRIksEMK702YHvI,1636
+hydroserverpy/api/models/etl/transformers/json_transformer.py,sha256=puKQI8abWJEQTcw34lEHgHjuPOuzcSBv95_txErzchk,2116
 hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
 hydroserverpy/api/models/iam/apikey.py,sha256=Z4iXg_K056naT3ogwc5wzyNnRpxHkOCz0lk-Gim4eL8,3146
@@ -61,9 +61,9 @@ hydroserverpy/api/services/sta/thing.py,sha256=Hyo3zTghSs7IIdsOGRu35i9w-aGOYlK9b
 hydroserverpy/api/services/sta/unit.py,sha256=NFToSAIGTwDfwYWe8Q-I_f5xsw_GYzFEkMnhSJ-ChvE,2178
 hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
 hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
-hydroserverpy-1.4.0b3.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
-hydroserverpy-1.4.0b3.dist-info/METADATA,sha256=NQ1axvaQ2Sh3BBMa4zK5TJZT10UO7aGuWRIH0BJf1i8,532
-hydroserverpy-1.4.0b3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hydroserverpy-1.4.0b3.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
-hydroserverpy-1.4.0b3.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-hydroserverpy-1.4.0b3.dist-info/RECORD,,
+hydroserverpy-1.4.0b4.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
+hydroserverpy-1.4.0b4.dist-info/METADATA,sha256=GCgulq3Im1uhFlRJtRg54dwOIqHK3wFfxAXpT6hlDlA,532
+hydroserverpy-1.4.0b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hydroserverpy-1.4.0b4.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
+hydroserverpy-1.4.0b4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+hydroserverpy-1.4.0b4.dist-info/RECORD,,