hydroserverpy 0.5.0b4__py3-none-any.whl → 0.5.0b5__py3-none-any.whl

hydroserverpy/api/models/sta/datastream.py

@@ -320,3 +320,23 @@ class Datastream(HydroServerModel, DatastreamFields):
             uid=self.uid,
             observations=observations,
         )
+
+    # TODO: Find a better long-term solution for this issue.
+    def sync_phenomenon_end_time(self):
+        """Ensures the phenomenon_end_time field matches the actual end time of the observations."""
+
+        response = self._connection.request(
+            "get", f"/api/data/datastreams/{str(self.uid)}/observations",
+            params={
+                "order": "desc",
+                "page": 1,
+                "page_size": 1
+            }
+        ).json()
+
+        if len(response["phenomenon_time"]) > 0:
+            self.phenomenon_end_time = datetime.fromisoformat(response["phenomenon_time"][0])
+        else:
+            self.phenomenon_end_time = None
+
+        self.save()

hydroserverpy/etl_csv/hydroserver_etl_csv.py

@@ -52,7 +52,7 @@ class HydroServerETLCSV:
         self._file_header_error = False
         self._file_timestamp_error = False
 
-        self._chunk_size = 10000
+        self._chunk_size = 1000
         self._observations = {}
 
     def run(self):
@@ -125,6 +125,9 @@ class HydroServerETLCSV:
         timestamp = self._parse_row_timestamp(row)
 
         for datastream in self._datastreams.values():
+            if index == self._data_source.settings["transformer"]["dataStartRow"]:
+                datastream.sync_phenomenon_end_time()
+
             if str(datastream.uid) not in self._datastream_start_row_indexes.keys():
                 if (
                     not datastream.phenomenon_end_time
@@ -296,8 +299,9 @@ class HydroServerETLCSV:
                         uid=datastream_id,
                         observations=observations_df,
                     )
-                except HTTPError:
+                except HTTPError as e:
                     failed_datastreams.append(datastream_id)
+                    logger.error(f"Failed to POST observations to datastream: {str(datastream_id)} - {e}")
 
             elif datastream_id in self._failed_datastreams:
                 logger.info(

{hydroserverpy-0.5.0b4.dist-info → hydroserverpy-0.5.0b5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 0.5.0b4
+Version: 0.5.0b5
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-0.5.0b4.dist-info → hydroserverpy-0.5.0b5.dist-info}/RECORD

@@ -15,7 +15,7 @@ hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F
 hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
 hydroserverpy/api/models/iam/workspace.py,sha256=s9u1oZyOdxM7txjJARFcIBrWMHQSDxODdreiatFsXJs,7331
 hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/api/models/sta/datastream.py,sha256=fvV4TPjLeThlH7kaYsLouwx74Ey4CCn2ixMPswULGwM,10497
+hydroserverpy/api/models/sta/datastream.py,sha256=vRjgwAKaoBJEtgUXrZjIS-VuIZsCilm7FRwbvLS8Y8o,11186
 hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
 hydroserverpy/api/models/sta/processing_level.py,sha256=y5_0wX7QGXgswvukXJtbpOiTCZ9pI8E08DXaTSUHakg,1470
 hydroserverpy/api/models/sta/result_qualifier.py,sha256=IJcY04KjP9e2D-jPzUJjH2PC-JvDNCjbi5LKkTVSwgw,1416
@@ -55,12 +55,12 @@ hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3
 hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
 hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
-hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=l1triLLJpXbOQF8PqueK6aRn2nMM_jZZY_hPpH5iJaE,14295
+hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=0ueBphEaAAlsb0cn71Ihgd5zOD8Zdu4Ts_yGwvXW53M,14544
 hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
 hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
-hydroserverpy-0.5.0b4.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
-hydroserverpy-0.5.0b4.dist-info/METADATA,sha256=oXiWKFvlOxc8GSnihwBPWNZb3QgpeF1dudmsql69WH4,532
-hydroserverpy-0.5.0b4.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-hydroserverpy-0.5.0b4.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
-hydroserverpy-0.5.0b4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-hydroserverpy-0.5.0b4.dist-info/RECORD,,
+hydroserverpy-0.5.0b5.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
+hydroserverpy-0.5.0b5.dist-info/METADATA,sha256=2MHOiPK4LfT1plINuLzDUiT0TwVmD2yqZu3edn0BYCI,532
+hydroserverpy-0.5.0b5.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+hydroserverpy-0.5.0b5.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
+hydroserverpy-0.5.0b5.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+hydroserverpy-0.5.0b5.dist-info/RECORD,,