hydroserverpy 0.5.0b2__py3-none-any.whl → 0.5.0b4__py3-none-any.whl

hydroserverpy/api/models/etl/data_source.py

@@ -1,8 +1,8 @@
+import requests
 import tempfile
 from typing import Union, List, Optional, TYPE_CHECKING
 from uuid import UUID
 from pydantic import BaseModel, Field
-from urllib.request import urlopen
 from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
 from .orchestration_system import OrchestrationSystem
 from .orchestration_configuration import OrchestrationConfigurationFields
@@ -135,14 +135,17 @@ class DataSource(HydroServerModel, DataSourceFields, OrchestrationConfigurationF
                 )
                 loader.run()
         elif self.settings["extractor"]["type"] == "HTTP":
-            with tempfile.NamedTemporaryFile(mode="w") as temp_file:
-                with urlopen(self.settings["extractor"]["urlTemplate"]) as response:
-                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
-                    while True:
-                        chunk = response.read(chunk_size)
-                        if not chunk:
-                            break
-                        temp_file.write(chunk)
+            with tempfile.NamedTemporaryFile(mode="w+") as temp_file:
+                response = requests.get(
+                    self.settings["extractor"]["urlTemplate"],
+                    stream=True,
+                    timeout=60,
+                )
+                response.raise_for_status()
+                chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        temp_file.write(chunk.decode("utf-8"))
                 temp_file.seek(0)
                 loader = HydroServerETLCSV(
                     self._connection, data_file=temp_file, data_source=self

hydroserverpy/api/models/sta/datastream.py

@@ -141,20 +141,6 @@ class Datastream(HydroServerModel, DatastreamFields):
             _connection=_connection, _model_ref="datastreams", _uid=_uid, **data
         )
 
-        self._workspace_id = str(
-            data.get("workspace_id")
-            or data.get("workspaceId")
-            or data["properties"]["workspace"]["id"]
-        )
-        self._processing_level_id = str(
-            data.get("workspace_id")
-            or data.get("workspaceId")
-            or data["properties"]["processingLevelId"]
-        )
-        self._unit_id = str(
-            data.get("unit_id") or data.get("unitId") or data["properties"]["unitId"]
-        )
-
         self._workspace = None
         self._thing = None
         self._observed_property = None
@@ -167,7 +153,8 @@ class Datastream(HydroServerModel, DatastreamFields):
         """The workspace this datastream belongs to."""
 
         if self._workspace is None:
-            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
+            self._workspace = self._connection.workspaces.get(uid=datastream["workspaceId"])
 
         return self._workspace
 
@@ -246,7 +233,8 @@ class Datastream(HydroServerModel, DatastreamFields):
         """The unit this datastream uses."""
 
         if self._unit is None:
-            self._unit = self._connection.units.get(uid=self._unit_id)
+            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
+            self._unit = self._connection.units.get(uid=datastream["unitId"])
             self._original_data["unit"] = self._unit
 
         return self._unit
@@ -263,9 +251,8 @@ class Datastream(HydroServerModel, DatastreamFields):
         """The processing level of this datastream."""
 
         if self._processing_level is None:
-            self._processing_level = self._connection.processinglevels.get(
-                uid=self._processing_level_id
-            )
+            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
+            self._processing_level = self._connection.processinglevels.get(uid=datastream["processingLevelId"])
             self._original_data["processing_level"] = self._processing_level
 
         return self._processing_level
@@ -285,13 +272,10 @@ class Datastream(HydroServerModel, DatastreamFields):
         """Refresh this datastream from HydroServer."""
 
         self._workspace = None
-        self._workspace_id = None
         self._thing = None
         self._observed_property = None
         self._unit = None
-        self._unit_id = None
         self._processing_level = None
-        self._processing_level_id = None
         self._sensor = None
         super()._refresh()
 

hydroserverpy/etl_csv/hydroserver_etl_csv.py

@@ -1,4 +1,5 @@
 import csv
+import math
 import logging
 import croniter
 import pandas as pd
@@ -138,14 +139,27 @@ class HydroServerETLCSV:
                 if str(datastream.uid) not in self._observations.keys():
                     self._observations[str(datastream.uid)] = []
 
+                raw_result = row[
+                    self._datastream_column_indexes[
+                        self._datastream_mapping[str(datastream.uid)]
+                    ]
+                ]
+
+                if isinstance(raw_result, (int, float)):
+                    result = raw_result
+                else:
+                    try:
+                        result = float(raw_result)
+                    except (TypeError, ValueError):
+                        result = datastream.no_data_value
+
+                if math.isnan(result):
+                    result = datastream.no_data_value
+
                 self._observations[str(datastream.uid)].append(
                     {
                         "phenomenon_time": timestamp,
-                        "result": row[
-                            self._datastream_column_indexes[
-                                self._datastream_mapping[str(datastream.uid)]
-                            ]
-                        ],
+                        "result": result,
                     }
                 )
 
@@ -200,8 +214,7 @@ class HydroServerETLCSV:
 
         try:
             if (
-                self._data_source.settings["transformer"].get("timestampFormat")
-                == "ISO8601"
+                self._data_source.settings["transformer"].get("timestampFormat") in ["utc", "constant", "ISO8601"]
                 or self._data_source.settings["transformer"].get("timestampFormat")
                 is None
             ):
@@ -228,11 +241,11 @@ class HydroServerETLCSV:
                     timestamp = timestamp.replace(
                         tzinfo=datetime.strptime(
                             self._data_source.settings["transformer"].get(
-                                "timestampFormat"
+                                "timestampOffset"
                             )[:-2]
                             + ":"
                             + self._data_source.settings["transformer"].get(
-                                "timestampFormat"
+                                "timestampOffset"
                             )[3:],
                             "%z",
                         ).tzinfo
@@ -309,13 +322,13 @@ class HydroServerETLCSV:
 
         if self._data_source.crontab is not None:
             next_run = croniter.croniter(
-                self._data_source.crontab, datetime.now()
+                self._data_source.crontab, datetime.now(timezone.utc)
             ).get_next(datetime)
         elif (
             self._data_source.interval is not None
             and self._data_source.interval_units is not None
         ):
-            next_run = datetime.now() + timedelta(
+            next_run = datetime.now(timezone.utc) + timedelta(
                 **{self._data_source.interval_units: self._data_source.interval}
             )
         else:

{hydroserverpy-0.5.0b2.dist-info → hydroserverpy-0.5.0b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 0.5.0b2
+Version: 0.5.0b4
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-0.5.0b2.dist-info → hydroserverpy-0.5.0b4.dist-info}/RECORD

@@ -6,7 +6,7 @@ hydroserverpy/api/models/__init__.py,sha256=ELrf3b7Aix7YcVF__Q_8e_G_FF8GYlX0J5l7
 hydroserverpy/api/models/base.py,sha256=dc2tfMSgizymxAAOVURfy7Jzeh6xIiiq7hfWZI7l1_Q,2280
 hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/api/models/etl/data_archive.py,sha256=u-gpvUsaWaw0kyF3bPMm2e55Jx2yhvSV9ufXXaNtrTc,3429
-hydroserverpy/api/models/etl/data_source.py,sha256=ca-9KKVhkLNaUn3vOIk-JgdWk58fTRme8YKIesk8WIw,5455
+hydroserverpy/api/models/etl/data_source.py,sha256=x88Z-CAzDljQEbGlif1KmQ4zNjQBiJrKNqirpoCZnPs,5538
 hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJFJg6aGogW5ZZk7fA17y4p--yWwiOhZ0,1367
 hydroserverpy/api/models/etl/orchestration_system.py,sha256=25En2G0z1gQzN-RW3UlrEGgkC952QDW21oYnawCX8hY,2357
 hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,7 +15,7 @@ hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F
 hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
 hydroserverpy/api/models/iam/workspace.py,sha256=s9u1oZyOdxM7txjJARFcIBrWMHQSDxODdreiatFsXJs,7331
 hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydroserverpy/api/models/sta/datastream.py,sha256=sB-KifvegbyDUnyPE_NCHFrab1ZSVVb6g-Gs7kUgMiE,10774
+hydroserverpy/api/models/sta/datastream.py,sha256=fvV4TPjLeThlH7kaYsLouwx74Ey4CCn2ixMPswULGwM,10497
 hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
 hydroserverpy/api/models/sta/processing_level.py,sha256=y5_0wX7QGXgswvukXJtbpOiTCZ9pI8E08DXaTSUHakg,1470
 hydroserverpy/api/models/sta/result_qualifier.py,sha256=IJcY04KjP9e2D-jPzUJjH2PC-JvDNCjbi5LKkTVSwgw,1416
@@ -55,12 +55,12 @@ hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3
 hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
 hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
-hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=1RRqZWXb8pMhkeg6Tn1IbQ8SdKNgOl78eqeBguEXnog,13849
+hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=l1triLLJpXbOQF8PqueK6aRn2nMM_jZZY_hPpH5iJaE,14295
 hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
 hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
-hydroserverpy-0.5.0b2.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
-hydroserverpy-0.5.0b2.dist-info/METADATA,sha256=IBrwK9SbCsLvo0JQyIEQmBCxSMpdkZ4i0uNTTVU96cY,532
-hydroserverpy-0.5.0b2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-hydroserverpy-0.5.0b2.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
-hydroserverpy-0.5.0b2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-hydroserverpy-0.5.0b2.dist-info/RECORD,,
+hydroserverpy-0.5.0b4.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
+hydroserverpy-0.5.0b4.dist-info/METADATA,sha256=oXiWKFvlOxc8GSnihwBPWNZb3QgpeF1dudmsql69WH4,532
+hydroserverpy-0.5.0b4.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+hydroserverpy-0.5.0b4.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
+hydroserverpy-0.5.0b4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+hydroserverpy-0.5.0b4.dist-info/RECORD,,

{hydroserverpy-0.5.0b2.dist-info → hydroserverpy-0.5.0b4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any