hydroserverpy 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

hydroserverpy/core/schemas/data_sources.py

@@ -6,7 +6,7 @@ from datetime import datetime
 from uuid import UUID
 from urllib.request import urlopen
 from hydroserverpy.core.schemas.base import HydroServerCoreModel
-from hydroserverpy.etl.service import HydroServerETL
+from hydroserverpy.etl_csv.hydroserver_etl_csv import HydroServerETLCSV
 
 if TYPE_CHECKING:
     from hydroserverpy.core.schemas.data_loaders import DataLoader
@@ -15,92 +15,105 @@ if TYPE_CHECKING:
 
 class DataSourceFields(BaseModel):
     name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the data source.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The name of the data source.",
     )
     path: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The path to a local data source file.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The path to a local data source file.",
     )
     link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The link to a remote data source file.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The link to a remote data source file.",
     )
     header_row: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The row number where the data begins.'
+        None, gt=0, lt=9999, description="The row number where the data begins."
     )
     data_start_row: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The row number where the data begins.'
+        None, gt=0, lt=9999, description="The row number where the data begins."
     )
     delimiter: Optional[str] = Field(
-        ',', strip_whitespace=True, max_length=1,
-        description='The delimiter used by the data source file.'
+        ",",
+        strip_whitespace=True,
+        max_length=1,
+        description="The delimiter used by the data source file.",
     )
     quote_char: Optional[str] = Field(
-        '"', strip_whitespace=True, max_length=1,
-        description='The quote delimiter character used by the data source file.'
+        '"',
+        strip_whitespace=True,
+        max_length=1,
+        description="The quote delimiter character used by the data source file.",
     )
     interval: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The time interval at which the data source should be loaded.'
-    )
-    interval_units: Optional[Literal['minutes', 'hours', 'days', 'weeks', 'months']] = Field(
         None,
-        description='The interval units used by the data source file.'
+        gt=0,
+        lt=9999,
+        description="The time interval at which the data source should be loaded.",
+    )
+    interval_units: Optional[Literal["minutes", "hours", "days", "weeks", "months"]] = (
+        Field(None, description="The interval units used by the data source file.")
     )
     crontab: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The crontab used to schedule when the data source should be loaded.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The crontab used to schedule when the data source should be loaded.",
     )
     start_time: Optional[datetime] = Field(
-        None,
-        description='When the data source should begin being loaded.'
+        None, description="When the data source should begin being loaded."
     )
     end_time: Optional[datetime] = Field(
-        None,
-        description='When the data source should stop being loaded.'
+        None, description="When the data source should stop being loaded."
     )
     paused: Optional[bool] = Field(
-        False,
-        description='Whether loading the data source should be paused or not.'
+        False, description="Whether loading the data source should be paused or not."
     )
     timestamp_column: Union[int, str] = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The column of the data source file containing the timestamps.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The column of the data source file containing the timestamps.",
     )
     timestamp_format: Optional[str] = Field(
-        '%Y-%m-%dT%H:%M:%S%Z', strip_whitespace=True, max_length=255,
-        description='The format of the timestamps, using Python\'s datetime strftime codes.'
+        "%Y-%m-%dT%H:%M:%S%Z",
+        strip_whitespace=True,
+        max_length=255,
+        description="The format of the timestamps, using Python's datetime strftime codes.",
     )
     timestamp_offset: Optional[str] = Field(
-        '+0000', strip_whitespace=True, max_length=255,
-        description='An ISO 8601 time zone offset designator code to be applied to timestamps in the data source file.'
+        "+0000",
+        strip_whitespace=True,
+        max_length=255,
+        description="An ISO 8601 time zone offset designator code to be applied to timestamps in the data source file.",
     )
     data_loader_id: UUID = Field(
         ...,
-        description='The ID of the data loader responsible for loading this data source.'
+        description="The ID of the data loader responsible for loading this data source.",
     )
     data_source_thru: Optional[datetime] = Field(
-        None,
-        description='The timestamp through which the data source contains data.'
+        None, description="The timestamp through which the data source contains data."
     )
     last_sync_successful: Optional[bool] = Field(
-        None,
-        description='Whether the last data loading attempt was successful of not.'
+        None, description="Whether the last data loading attempt was successful of not."
     )
     last_sync_message: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A message generated by the data loader it attempted to load data from this data source.'
+        None,
+        strip_whitespace=True,
+        description="A message generated by the data loader it attempted to load data from this data source.",
     )
     last_synced: Optional[datetime] = Field(
         None,
-        description='The last time the data loader attempted to load data from this data source.'
+        description="The last time the data loader attempted to load data from this data source.",
     )
     next_sync: Optional[datetime] = Field(
         None,
-        description="The next time the data loader will attempt to load data from this data source."
+        description="The next time the data loader will attempt to load data from this data source.",
     )
 
 
@@ -128,7 +141,7 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
         self._data_loader = None
 
     @property
-    def datastreams(self) -> List['Datastream']:
+    def datastreams(self) -> List["Datastream"]:
         """
         Retrieve the datastreams associated with the DataSource. If not already cached, fetch the datastreams from the
         server.
@@ -143,7 +156,7 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
         return self._datastreams
 
     @property
-    def data_loader(self) -> 'DataLoader':
+    def data_loader(self) -> "DataLoader":
         """
         Retrieve the data loader associated with the data source. If not already cached, fetch the data loader from the
         server.
@@ -153,7 +166,9 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
         """
 
         if self._data_loader is None:
-            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
+            self._data_loader = self._endpoint._service.dataloaders.get(
+                uid=self.data_loader_id
+            )  # noqa
 
         return self._data_loader
 
@@ -163,13 +178,15 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
         they were previously loaded.
         """
 
-        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=["uid"])
         self._original_data = entity
         self.__dict__.update(entity)
         if self._datastreams is not None:
             self._datastreams = self._endpoint.list_datastreams(uid=self.uid)
         if self._data_loader is not None:
-            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
+            self._data_loader = self._endpoint._service.dataloaders.get(
+                uid=self.data_loader_id
+            )  # noqa
 
     def load_observations(self) -> None:
         """
@@ -177,16 +194,16 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
         """
 
         if self.path:
-            with open(self.path, 'rb') as f:
-                with io.TextIOWrapper(f, encoding='utf-8') as data_file:
-                    hs_etl = HydroServerETL(
-                        service=getattr(self._endpoint, '_service'),
+            with open(self.path, "rb") as f:
+                with io.TextIOWrapper(f, encoding="utf-8") as data_file:
+                    hs_etl = HydroServerETLCSV(
+                        service=getattr(self._endpoint, "_service"),
                         data_file=data_file,
                         data_source=self,
                     )
                     hs_etl.run()
         elif self.link:
-            with tempfile.NamedTemporaryFile(mode='w+b') as temp_file:
+            with tempfile.NamedTemporaryFile(mode="w+b") as temp_file:
                 with urlopen(self.link) as response:
                     chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
                     while True:
@@ -195,9 +212,9 @@ class DataSource(HydroServerCoreModel, DataSourceFields):
                             break
                         temp_file.write(chunk)
                 temp_file.seek(0)
-                with io.TextIOWrapper(temp_file, encoding='utf-8') as data_file:
-                    hs_etl = HydroServerETL(
-                        service=getattr(self._endpoint, '_service'),
+                with io.TextIOWrapper(temp_file, encoding="utf-8") as data_file:
+                    hs_etl = HydroServerETLCSV(
+                        service=getattr(self._endpoint, "_service"),
                         data_file=data_file,
                         data_source=self,
                     )

hydroserverpy/core/schemas/datastreams.py

@@ -16,108 +16,118 @@ if TYPE_CHECKING:
 
 class DatastreamFields(BaseModel):
     name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the datastream.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The name of the datastream.",
     )
     description: str = Field(
-        ..., strip_whitespace=True,
-        description='A description of the datastream.'
+        ..., strip_whitespace=True, description="A description of the datastream."
     )
     observation_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The type of observation recorded in this datastream'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The type of observation recorded in this datastream",
     )
     sampled_medium: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The physical medium in which the observations were sampled.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The physical medium in which the observations were sampled.",
     )
     no_data_value: float = Field(
         ...,
-        description='A numerical value representing no data at a given timestamp.',
+        description="A numerical value representing no data at a given timestamp.",
     )
     aggregation_statistic: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The statistic calculated over the time aggregation interval of observations in this datastream.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The statistic calculated over the time aggregation interval of observations in this datastream.",
     )
     time_aggregation_interval: float = Field(
         ...,
-        description='The time interval over which the aggregation statistic is applied to observations.',
+        description="The time interval over which the aggregation statistic is applied to observations.",
     )
     status: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The current status of this datastream.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The current status of this datastream.",
     )
     result_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The type of result recorded in this datastream.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The type of result recorded in this datastream.",
     )
     value_count: Optional[int] = Field(
-        None, ge=0,
-        description='The total number of observations in this datastream.'
+        None, ge=0, description="The total number of observations in this datastream."
     )
     phenomenon_begin_time: Optional[datetime] = Field(
         None,
-        description='The timestamp representing when the first phenomenon recorded in this datastream occurred.'
+        description="The timestamp representing when the first phenomenon recorded in this datastream occurred.",
     )
     phenomenon_end_time: Optional[datetime] = Field(
         None,
-        description='The timestamp representing when the last phenomenon recorded in this datastream occurred.'
+        description="The timestamp representing when the last phenomenon recorded in this datastream occurred.",
     )
     result_begin_time: Optional[datetime] = Field(
         None,
-        description='The timestamp representing when the first observation of this datastream was recorded.'
+        description="The timestamp representing when the first observation of this datastream was recorded.",
     )
     result_end_time: Optional[datetime] = Field(
         None,
-        description='The timestamp representing when the last observation of this datastream was recorded.'
+        description="The timestamp representing when the last observation of this datastream was recorded.",
     )
     data_source_id: Optional[UUID] = Field(
-        None,
-        description='The data source for observations of this datastream.'
+        None, description="The data source for observations of this datastream."
     )
     data_source_column: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The name of the column containing this datastream\'s observations in the data source file.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The name of the column containing this datastream's observations in the data source file.",
     )
     is_visible: bool = Field(
-        True,
-        description='Whether this datastream is publicly visible.'
+        True, description="Whether this datastream is publicly visible."
     )
     is_data_visible: bool = Field(
         True,
-        description='Whether this observations associated with this datastream are publicly visible.'
+        description="Whether this observations associated with this datastream are publicly visible.",
     )
     thing_id: UUID = Field(
         ...,
-        description='The site/thing from which observations of this datastream were recorded.'
+        description="The site/thing from which observations of this datastream were recorded.",
     )
     sensor_id: UUID = Field(
-        ...,
-        description='The sensor used to record observations of this datastream.'
+        ..., description="The sensor used to record observations of this datastream."
     )
     observed_property_id: UUID = Field(
-        ...,
-        description='The physical property being observed for this datastream.'
+        ..., description="The physical property being observed for this datastream."
     )
     processing_level_id: UUID = Field(
-        ...,
-        description='The processing level applied to this datastream.'
+        ..., description="The processing level applied to this datastream."
     )
     unit_id: UUID = Field(
-        ...,
-        description='The unit used to record observations for this datastream.'
+        ..., description="The unit used to record observations for this datastream."
     )
-    time_aggregation_interval_units: Literal['seconds', 'minutes', 'hours', 'days'] = Field(
-        ...,
-        description='The time unit for this datastream\'s time aggregation interval'
+    time_aggregation_interval_units: Literal["seconds", "minutes", "hours", "days"] = (
+        Field(
+            ...,
+            description="The time unit for this datastream's time aggregation interval",
+        )
     )
     intended_time_spacing: Optional[float] = Field(
         None,
-        description='The time interval at which observations should be made for this datastream.'
+        description="The time interval at which observations should be made for this datastream.",
     )
-    intended_time_spacing_units: Optional[Literal['seconds', 'minutes', 'hours', 'days']] = Field(
+    intended_time_spacing_units: Optional[
+        Literal["seconds", "minutes", "hours", "days"]
+    ] = Field(
         None,
-        description='The time unit for this datastream\'s intended time spacing interval'
+        description="The time unit for this datastream's intended time spacing interval",
     )
 
 
@@ -153,7 +163,7 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         self._sensor = None
 
     @property
-    def thing(self) -> 'Thing':
+    def thing(self) -> "Thing":
         """
         The thing entity associated with the datastream. If not already cached, fetch it from the server.
 
@@ -167,7 +177,7 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         return self._thing
 
     @property
-    def data_source(self) -> 'DataSource':
+    def data_source(self) -> "DataSource":
         """
         The data source entity associated with the datastream. If not already cached, fetch it from the server.
 
@@ -176,12 +186,14 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         """
 
         if self._data_source is None:
-            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
+            self._data_source = self._endpoint._service.datasources.get(
+                uid=self.data_source_id
+            )  # noqa
 
         return self._data_source
 
     @property
-    def observed_property(self) -> 'ObservedProperty':
+    def observed_property(self) -> "ObservedProperty":
         """
         Retrieve the observed property entity associated with the datastream. If not already cached, fetch it from the
         server.
@@ -191,12 +203,14 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         """
 
         if self._observed_property is None:
-            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
+            self._observed_property = self._endpoint._service.observedproperties.get(
+                uid=self.observed_property_id
+            )  # noqa
 
         return self._observed_property
 
     @property
-    def processing_level(self) -> 'ProcessingLevel':
+    def processing_level(self) -> "ProcessingLevel":
         """
         Retrieve the processing level entity associated with the datastream. If not already cached, fetch it from the
         server.
@@ -206,12 +220,14 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         """
 
         if self._processing_level is None:
-            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
+            self._processing_level = self._endpoint._service.processinglevels.get(
+                uid=self.processing_level_id
+            )  # noqa
 
         return self._processing_level
 
     @property
-    def unit(self) -> 'Unit':
+    def unit(self) -> "Unit":
         """
         Retrieve the unit entity associated with the datastream. If not already cached, fetch it from the server.
 
@@ -225,7 +241,7 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         return self._unit
 
     @property
-    def sensor(self) -> 'Sensor':
+    def sensor(self) -> "Sensor":
         """
         Retrieve the sensor entity associated with the datastream. If not already cached, fetch it from the server.
 
@@ -234,7 +250,9 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         """
 
         if self._sensor is None:
-            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
+            self._sensor = self._endpoint._service.sensors.get(
+                uid=self.sensor_id
+            )  # noqa
 
         return self._sensor
 
@@ -244,30 +262,38 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         loaded.
         """
 
-        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=["uid"])
         self._original_data = entity
         self.__dict__.update(entity)
         if self._thing is not None:
             self._thing = self._endpoint._service.things.get(uid=self.thing_id)  # noqa
         if self._data_source is not None:
-            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
+            self._data_source = self._endpoint._service.datasources.get(
+                uid=self.data_source_id
+            )  # noqa
         if self._observed_property is not None:
-            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
+            self._observed_property = self._endpoint._service.observedproperties.get(
+                uid=self.observed_property_id
+            )  # noqa
         if self._processing_level is not None:
-            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
+            self._processing_level = self._endpoint._service.processinglevels.get(
+                uid=self.processing_level_id
+            )  # noqa
         if self._unit is not None:
             self._unit = self._endpoint._service.units.get(uid=self.unit_id)  # noqa
         if self._sensor is not None:
-            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
+            self._sensor = self._endpoint._service.sensors.get(
+                uid=self.sensor_id
+            )  # noqa
 
     def get_observations(
-            self,
-            start_time: datetime = None,
-            end_time: datetime = None,
-            page: int = 1,
-            page_size: int = 100000,
-            include_quality: bool = False,
-            fetch_all: bool = False
+        self,
+        start_time: datetime = None,
+        end_time: datetime = None,
+        page: int = 1,
+        page_size: int = 100000,
+        include_quality: bool = False,
+        fetch_all: bool = False,
     ) -> DataFrame:
         """
         Retrieve the observations for this datastream.
@@ -277,13 +303,18 @@ class Datastream(HydroServerCoreModel, DatastreamFields):
         """
 
         return self._endpoint.get_observations(
-            uid=self.uid, start_time=start_time, end_time=end_time, page=page, page_size=page_size,
-            include_quality=include_quality, fetch_all=fetch_all
+            uid=self.uid,
+            start_time=start_time,
+            end_time=end_time,
+            page=page,
+            page_size=page_size,
+            include_quality=include_quality,
+            fetch_all=fetch_all,
         )
 
     def load_observations(
-            self,
-            observations: DataFrame,
+        self,
+        observations: DataFrame,
     ) -> None:
         """
         Load a DataFrame of observations to the datastream.

hydroserverpy/core/schemas/observed_properties.py

@@ -5,24 +5,32 @@ from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
 class ObservedPropertyFields(BaseModel):
     name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the observed property.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The name of the observed property.",
     )
     definition: str = Field(
-        ..., strip_whitespace=True,
-        description='The definition of the observed property.'
+        ...,
+        strip_whitespace=True,
+        description="The definition of the observed property.",
     )
     description: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A description of the observed property.'
+        None,
+        strip_whitespace=True,
+        description="A description of the observed property.",
     )
     type: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The type of the observed property.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The type of the observed property.",
     )
     code: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='A code representing the observed property.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="A code representing the observed property.",
     )
 
 

hydroserverpy/core/schemas/processing_levels.py

@@ -5,16 +5,20 @@ from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
 class ProcessingLevelFields(BaseModel):
     code: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='A code representing the processing level.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="A code representing the processing level.",
     )
     definition: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='The definition of the processing level.'
+        None,
+        strip_whitespace=True,
+        description="The definition of the processing level.",
     )
     explanation: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='The explanation of the processing level.'
+        None,
+        strip_whitespace=True,
+        description="The explanation of the processing level.",
     )
 
 

hydroserverpy/core/schemas/result_qualifiers.py

@@ -5,12 +5,15 @@ from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
 class ResultQualifierFields(BaseModel):
     code: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='A code representing the result qualifier.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="A code representing the result qualifier.",
     )
     description: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A description of the result qualifier.'
+        None,
+        strip_whitespace=True,
+        description="A description of the result qualifier.",
     )