hydroserverpy 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

hydroserverpy/__init__.py

@@ -1,6 +1,6 @@
 from .core.service import HydroServer
 from .quality.service import HydroServerQualityControl
-from .etl.service import HydroServerETL
+from .etl.hydroserver_etl import HydroServerETL
 
 
 __all__ = [

hydroserverpy/core/endpoints/base.py

@@ -6,23 +6,26 @@ if TYPE_CHECKING:
     from hydroserverpy import HydroServer
     from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
-    HydroServerModelType = TypeVar('HydroServerModelType', bound=HydroServerCoreModel)
+    HydroServerModelType = TypeVar("HydroServerModelType", bound=HydroServerCoreModel)
 
 
-def expand_docstring(model: Optional[Type['HydroServerCoreModel']] = None, include_uid: bool = False):
+def expand_docstring(
+    model: Optional[Type["HydroServerCoreModel"]] = None, include_uid: bool = False
+):
     def decorator(func):
         docstring = func.__doc__
         if model is not None or include_uid is True:
-            docstring += '\n'
+            docstring += "\n"
         if include_uid is True:
-            docstring += f':param uid: The entity ID.\n'
-            docstring += f':type uid: Union[UUID, str]\n'
+            docstring += f":param uid: The entity ID.\n"
+            docstring += f":type uid: Union[UUID, str]\n"
         if model is not None:
             for field_name, field in model.model_fields.items():
-                docstring += f':param {field_name}: {field.description}\n'
+                docstring += f":param {field_name}: {field.description}\n"
                 docstring += f':type {field_name}: {getattr(field.annotation, "__name__", field.annotation)}\n'
         func.__doc__ = docstring
         return func
+
     return decorator
 
 
@@ -35,11 +38,11 @@ class HydroServerEndpoint:
     :ivar _endpoint_route: The specific route of the endpoint.
     """
 
-    _model: Type['HydroServerCoreModel']
+    _model: Type["HydroServerCoreModel"]
     _api_route: str
     _endpoint_route: str
 
-    def __init__(self, service: 'HydroServer') -> None:
+    def __init__(self, service: "HydroServer") -> None:
         """
         Initialize the HydroServerEndpoint.
 
@@ -49,9 +52,9 @@ class HydroServerEndpoint:
 
         self._service = service
 
-    def _get(self, uid: Optional[Union[UUID, str]] = None, params: dict = None) -> Union[
-        List['HydroServerModelType'], 'HydroServerModelType'
-    ]:
+    def _get(
+        self, uid: Optional[Union[UUID, str]] = None, params: dict = None
+    ) -> Union[List["HydroServerModelType"], "HydroServerModelType"]:
         """
         Fetch an entity collection or single entity from a HydroServer endpoint.
 
@@ -65,20 +68,22 @@ class HydroServerEndpoint:
             params = {}
 
         path = f'{self._api_route}/data/{self._endpoint_route}{"/" + str(uid) if uid else ""}'
-        response = getattr(self._service, '_request')('get', path, params=params)
+        response = getattr(self._service, "_request")("get", path, params=params)
 
         if uid:
             entity = json.loads(response.content)
-            result = self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+            result = self._model(
+                _endpoint=self, _uid=UUID(str(entity.pop("id"))), **entity
+            )
         else:
             result = [
-                self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+                self._model(_endpoint=self, _uid=UUID(str(entity.pop("id"))), **entity)
                 for entity in json.loads(response.content)
             ]
 
         return result
 
-    def _post(self, **kwargs) -> 'HydroServerModelType':
+    def _post(self, **kwargs) -> "HydroServerModelType":
         """
         Create a new entity using the endpoint.
 
@@ -87,16 +92,19 @@ class HydroServerEndpoint:
         :rtype: HydroServerModelType
         """
 
-        response = getattr(self._service, '_request')(
-            'post', f'{self._api_route}/data/{self._endpoint_route}',
-            headers={'Content-type': 'application/json'},
-            data=self._model(_endpoint=self, **kwargs).json(exclude_unset=True, by_alias=True),
+        response = getattr(self._service, "_request")(
+            "post",
+            f"{self._api_route}/data/{self._endpoint_route}",
+            headers={"Content-type": "application/json"},
+            data=self._model(_endpoint=self, **kwargs).json(
+                exclude_unset=True, by_alias=True
+            ),
         )
         entity = json.loads(response.content)
 
-        return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+        return self._model(_endpoint=self, _uid=UUID(str(entity.pop("id"))), **entity)
 
-    def _patch(self, uid: Union[UUID, str], **kwargs) -> 'HydroServerModelType':
+    def _patch(self, uid: Union[UUID, str], **kwargs) -> "HydroServerModelType":
         """
         Update an existing entity in the endpoint.
 
@@ -107,17 +115,21 @@ class HydroServerEndpoint:
         :rtype: HydroServerModelType
         """
 
-        response = getattr(self._service, '_request')(
-            'patch', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
-            headers={'Content-type': 'application/json'},
-            data=json.dumps({
-                self._model.model_fields[key].serialization_alias: value
-                for key, value in kwargs.items()
-            }, default=str)
+        response = getattr(self._service, "_request")(
+            "patch",
+            f"{self._api_route}/data/{self._endpoint_route}/{str(uid)}",
+            headers={"Content-type": "application/json"},
+            data=json.dumps(
+                {
+                    self._model.model_fields[key].serialization_alias: value
+                    for key, value in kwargs.items()
+                },
+                default=str,
+            ),
         )
         entity = json.loads(response.content)
 
-        return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+        return self._model(_endpoint=self, _uid=UUID(str(entity.pop("id"))), **entity)
 
     def _delete(self, uid: Union[UUID, str]) -> None:
         """
@@ -128,6 +140,7 @@ class HydroServerEndpoint:
         :returns: None
         """
 
-        getattr(self._service, '_request')(
-            'delete', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
+        getattr(self._service, "_request")(
+            "delete",
+            f"{self._api_route}/data/{self._endpoint_route}/{str(uid)}",
         )

hydroserverpy/core/endpoints/data_loaders.py

@@ -18,7 +18,7 @@ class DataLoaderEndpoint(HydroServerEndpoint):
     :ivar _endpoint_route: The specific route of the endpoint, set to `'data-loaders'`.
     """
 
-    def __init__(self, service: 'HydroServer') -> None:
+    def __init__(self, service: "HydroServer") -> None:
         """
         Initialize the DataLoaderEndpoint.
 
@@ -29,7 +29,7 @@ class DataLoaderEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = DataLoader
         self._api_route = self._service.api_route
-        self._endpoint_route = 'data-loaders'
+        self._endpoint_route = "data-loaders"
 
     def list(self) -> List[DataLoader]:
         """
@@ -80,13 +80,14 @@ class DataLoaderEndpoint(HydroServerEndpoint):
         :rtype: List[DataSource]
         """
 
-        response = getattr(self._service, '_request')(
-            'get', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}/data-sources'
+        response = getattr(self._service, "_request")(
+            "get",
+            f"{self._api_route}/data/{self._endpoint_route}/{str(uid)}/data-sources",
         )
 
         endpoint = DataSourceEndpoint(self._service)
 
         return [
-            DataSource(_endpoint=endpoint, _uid=UUID(str(entity.pop('id'))), **entity)
+            DataSource(_endpoint=endpoint, _uid=UUID(str(entity.pop("id"))), **entity)
             for entity in json.loads(response.content)
         ]

hydroserverpy/core/endpoints/data_sources.py

@@ -18,7 +18,7 @@ class DataSourceEndpoint(HydroServerEndpoint):
     :ivar _endpoint_route: The specific route of the endpoint, set to `'data-sources'`.
     """
 
-    def __init__(self, service: 'HydroServer') -> None:
+    def __init__(self, service: "HydroServer") -> None:
         """
         Initialize the DataSourceEndpoint.
 
@@ -29,7 +29,7 @@ class DataSourceEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = DataSource
         self._api_route = self._service.api_route
-        self._endpoint_route = 'data-sources'
+        self._endpoint_route = "data-sources"
 
     def list(self) -> List[DataSource]:
         """
@@ -80,13 +80,14 @@ class DataSourceEndpoint(HydroServerEndpoint):
         :rtype: List[Datastream]
         """
 
-        response = getattr(self._service, '_request')(
-            'get', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}/datastreams'
+        response = getattr(self._service, "_request")(
+            "get",
+            f"{self._api_route}/data/{self._endpoint_route}/{str(uid)}/datastreams",
         )
 
         endpoint = DatastreamEndpoint(self._service)
 
         return [
-            Datastream(_endpoint=endpoint, _uid=UUID(str(entity.pop('id'))), **entity)
+            Datastream(_endpoint=endpoint, _uid=UUID(str(entity.pop("id"))), **entity)
             for entity in json.loads(response.content)
         ]

hydroserverpy/core/endpoints/datastreams.py

@@ -30,9 +30,11 @@ class DatastreamEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = Datastream
         self._api_route = self._service.api_route
-        self._endpoint_route = 'datastreams'
+        self._endpoint_route = "datastreams"
 
-    def list(self, owned_only: bool = False, primary_owned_only: bool = False) -> List[Datastream]:
+    def list(
+        self, owned_only: bool = False, primary_owned_only: bool = False
+    ) -> List[Datastream]:
         """
         Retrieve a collection of datastreams owned by the logged-in user.
 
@@ -40,10 +42,12 @@ class DatastreamEndpoint(HydroServerEndpoint):
         :param primary_owned_only: Only list datastreams primary owned by the logged-in user.
         """
 
-        return super()._get(params={
-            'owned_only': owned_only,
-            'primary_owned_only': primary_owned_only,
-        })
+        return super()._get(
+            params={
+                "owned_only": owned_only,
+                "primary_owned_only": primary_owned_only,
+            }
+        )
 
     @expand_docstring(include_uid=True)
     def get(self, uid: Union[UUID, str]) -> Datastream:
@@ -78,14 +82,14 @@ class DatastreamEndpoint(HydroServerEndpoint):
         super()._delete(uid=uid)
 
     def get_observations(
-            self,
-            uid: Union[UUID, str],
-            start_time: datetime = None,
-            end_time: datetime = None,
-            page: int = 1,
-            page_size: int = 100000,
-            include_quality: bool = False,
-            fetch_all: bool = False
+        self,
+        uid: Union[UUID, str],
+        start_time: datetime = None,
+        end_time: datetime = None,
+        page: int = 1,
+        page_size: int = 100000,
+        include_quality: bool = False,
+        fetch_all: bool = False,
     ) -> pd.DataFrame:
         """
         Retrieve observations from a specific datastream.
@@ -110,9 +114,13 @@ class DatastreamEndpoint(HydroServerEndpoint):
 
         filters = []
         if start_time:
-            filters.append(f'phenomenonTime ge {start_time.strftime("%Y-%m-%dT%H:%M:%S%z")}')
+            filters.append(
+                f'phenomenonTime ge {start_time.strftime("%Y-%m-%dT%H:%M:%S%z")}'
+            )
         if end_time:
-            filters.append(f'phenomenonTime le {end_time.strftime("%Y-%m-%dT%H:%M:%S%z")}')
+            filters.append(
+                f'phenomenonTime le {end_time.strftime("%Y-%m-%dT%H:%M:%S%z")}'
+            )
 
         if fetch_all:
             page = 1
@@ -120,41 +128,60 @@ class DatastreamEndpoint(HydroServerEndpoint):
         observations = []
 
         while True:
-            response = getattr(self._service, '_request')(
-                'get', f'{self._api_route}/sensorthings/v1.1/Datastreams(\'{str(uid)}\')/Observations',
+            response = getattr(self._service, "_request")(
+                "get",
+                f"{self._api_route}/sensorthings/v1.1/Datastreams('{str(uid)}')/Observations",
                 params={
-                    '$resultFormat': 'dataArray',
-                    '$select': f'phenomenonTime,result{",resultQuality" if include_quality else ""}',
-                    '$count': True,
-                    '$top': page_size,
-                    '$skip': (page - 1) * page_size,
-                    '$filter': ' and '.join(filters) if filters else None
-                }
+                    "$resultFormat": "dataArray",
+                    "$select": f'phenomenonTime,result{",resultQuality" if include_quality else ""}',
+                    "$count": True,
+                    "$top": page_size,
+                    "$skip": (page - 1) * page_size,
+                    "$filter": " and ".join(filters) if filters else None,
+                },
             )
             response_content = json.loads(response.content)
-            data_array = response_content['value'][0]['dataArray'] if response_content['value'] else []
-            observations.extend([[
-                obs[0], obs[1],
-                obs[2]['qualityCode'] if obs[2]['qualityCode'] else None,
-                obs[2]['resultQualifiers'] if obs[2]['resultQualifiers'] else None
-            ] if include_quality else [obs[0], obs[1]] for obs in data_array])
+            data_array = (
+                response_content["value"][0]["dataArray"]
+                if response_content["value"]
+                else []
+            )
+            observations.extend(
+                [
+                    (
+                        [
+                            obs[0],
+                            obs[1],
+                            obs[2]["qualityCode"] if obs[2]["qualityCode"] else None,
+                            (
+                                obs[2]["resultQualifiers"]
+                                if obs[2]["resultQualifiers"]
+                                else None
+                            ),
+                        ]
+                        if include_quality
+                        else [obs[0], obs[1]]
+                    )
+                    for obs in data_array
+                ]
+            )
             if not fetch_all or len(data_array) < page_size:
                 break
             page += 1
 
-        columns = ['timestamp', 'value']
+        columns = ["timestamp", "value"]
         if include_quality:
-            columns.extend(['quality_code', 'result_quality'])
+            columns.extend(["quality_code", "result_quality"])
 
         data_frame = pd.DataFrame(observations, columns=columns)
-        data_frame['timestamp'] = pd.to_datetime(data_frame['timestamp'])
+        data_frame["timestamp"] = pd.to_datetime(data_frame["timestamp"])
 
         return data_frame
 
     def load_observations(
-            self,
-            uid: Union[UUID, str],
-            observations: pd.DataFrame,
+        self,
+        uid: Union[UUID, str],
+        observations: pd.DataFrame,
     ) -> None:
         """
         Load observations to a specific datastream.
@@ -168,21 +195,31 @@ class DatastreamEndpoint(HydroServerEndpoint):
 
         data_array = [
             [
-                row['timestamp'].strftime('%Y-%m-%dT%H:%M:%S%z'),
-                row['value'],
-                {
-                    'qualityCode': row.get('quality_code', None),
-                    'resultQualifiers': row.get('result_qualifiers', []),
-                } if 'quality_code' in row or 'result_qualifiers' in row else {}
-            ] for _, row in observations.iterrows()
+                row["timestamp"].strftime("%Y-%m-%dT%H:%M:%S%z"),
+                row["value"],
+                (
+                    {
+                        "qualityCode": row.get("quality_code", None),
+                        "resultQualifiers": row.get("result_qualifiers", []),
+                    }
+                    if "quality_code" in row or "result_qualifiers" in row
+                    else {}
+                ),
+            ]
+            for _, row in observations.iterrows()
         ]
 
-        getattr(self._service, '_request')(
-            'post', f'{self._api_route}/sensorthings/v1.1/CreateObservations',
-            headers={'Content-type': 'application/json'},
-            data=json.dumps([{
-                'Datastream': {'@iot.id': str(uid)},
-                'components': ['phenomenonTime', 'result', 'resultQuality'],
-                'dataArray': data_array
-            }]),
+        getattr(self._service, "_request")(
+            "post",
+            f"{self._api_route}/sensorthings/v1.1/CreateObservations",
+            headers={"Content-type": "application/json"},
+            data=json.dumps(
+                [
+                    {
+                        "Datastream": {"@iot.id": str(uid)},
+                        "components": ["phenomenonTime", "result", "resultQuality"],
+                        "dataArray": data_array,
+                    }
+                ]
+            ),
         )

hydroserverpy/core/endpoints/observed_properties.py

@@ -27,13 +27,13 @@ class ObservedPropertyEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = ObservedProperty
         self._api_route = self._service.api_route
-        self._endpoint_route = 'observed-properties'
+        self._endpoint_route = "observed-properties"
 
     def list(
-            self,
-            include_owned: bool = True,
-            include_unowned: bool = True,
-            include_templates: bool = True
+        self,
+        include_owned: bool = True,
+        include_unowned: bool = True,
+        include_templates: bool = True,
     ) -> List[ObservedProperty]:
         """
         Retrieve a collection of observed properties.
@@ -43,22 +43,40 @@ class ObservedPropertyEndpoint(HydroServerEndpoint):
         :param include_templates: Whether to include template observed properties.
         """
 
-        if include_owned is True and include_unowned is True and include_templates is True:
-            owner = 'anyUserOrNoUser'
-        elif include_owned is True and include_unowned is True and include_templates is False:
-            owner = 'anyUser'
-        elif include_owned is True and include_unowned is False and include_templates is True:
-            owner = 'currentUserOrNoUser'
-        elif include_owned is True and include_unowned is False and include_templates is False:
-            owner = 'currentUser'
-        elif include_owned is False and include_unowned is False and include_templates is True:
-            owner = 'noUser'
+        if (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is True
+        ):
+            owner = "anyUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is False
+        ):
+            owner = "anyUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "currentUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is False
+        ):
+            owner = "currentUser"
+        elif (
+            include_owned is False
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "noUser"
         else:
             return []
 
-        return super()._get(params={
-            'owner': owner
-        })
+        return super()._get(params={"owner": owner})
 
     @expand_docstring(include_uid=True)
     def get(self, uid: Union[UUID, str]) -> ObservedProperty:

hydroserverpy/core/endpoints/processing_levels.py

@@ -27,13 +27,13 @@ class ProcessingLevelEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = ProcessingLevel
         self._api_route = self._service.api_route
-        self._endpoint_route = 'processing-levels'
+        self._endpoint_route = "processing-levels"
 
     def list(
-            self,
-            include_owned: bool = True,
-            include_unowned: bool = True,
-            include_templates: bool = True
+        self,
+        include_owned: bool = True,
+        include_unowned: bool = True,
+        include_templates: bool = True,
     ) -> List[ProcessingLevel]:
         """
         Retrieve a collection of processing levels.
@@ -43,22 +43,40 @@ class ProcessingLevelEndpoint(HydroServerEndpoint):
         :param include_templates: Whether to include template observed properties.
         """
 
-        if include_owned is True and include_unowned is True and include_templates is True:
-            owner = 'anyUserOrNoUser'
-        elif include_owned is True and include_unowned is True and include_templates is False:
-            owner = 'anyUser'
-        elif include_owned is True and include_unowned is False and include_templates is True:
-            owner = 'currentUserOrNoUser'
-        elif include_owned is True and include_unowned is False and include_templates is False:
-            owner = 'currentUser'
-        elif include_owned is False and include_unowned is False and include_templates is True:
-            owner = 'noUser'
+        if (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is True
+        ):
+            owner = "anyUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is False
+        ):
+            owner = "anyUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "currentUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is False
+        ):
+            owner = "currentUser"
+        elif (
+            include_owned is False
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "noUser"
         else:
             return []
 
-        return super()._get(params={
-            'owner': owner
-        })
+        return super()._get(params={"owner": owner})
 
     @expand_docstring(include_uid=True)
     def get(self, uid: Union[UUID, str]) -> ProcessingLevel:

hydroserverpy/core/endpoints/result_qualifiers.py

@@ -16,7 +16,7 @@ class ResultQualifierEndpoint(HydroServerEndpoint):
     :ivar _endpoint_route: The specific route of the endpoint, set to `'result-qualifiers'`.
     """
 
-    def __init__(self, service: 'HydroServer'):
+    def __init__(self, service: "HydroServer"):
         """
         Initialize the ResultQualifierEndpoint.
 
@@ -27,13 +27,13 @@ class ResultQualifierEndpoint(HydroServerEndpoint):
         super().__init__(service)
         self._model = ResultQualifier
         self._api_route = self._service.api_route
-        self._endpoint_route = 'result-qualifiers'
+        self._endpoint_route = "result-qualifiers"
 
     def list(
-            self,
-            include_owned: bool = True,
-            include_unowned: bool = True,
-            include_templates: bool = True
+        self,
+        include_owned: bool = True,
+        include_unowned: bool = True,
+        include_templates: bool = True,
     ) -> List[ResultQualifier]:
         """
         Retrieve a collection of result qualifiers.
@@ -43,22 +43,40 @@ class ResultQualifierEndpoint(HydroServerEndpoint):
         :param include_templates: Whether to include template observed properties.
         """
 
-        if include_owned is True and include_unowned is True and include_templates is True:
-            owner = 'anyUserOrNoUser'
-        elif include_owned is True and include_unowned is True and include_templates is False:
-            owner = 'anyUser'
-        elif include_owned is True and include_unowned is False and include_templates is True:
-            owner = 'currentUserOrNoUser'
-        elif include_owned is True and include_unowned is False and include_templates is False:
-            owner = 'currentUser'
-        elif include_owned is False and include_unowned is False and include_templates is True:
-            owner = 'noUser'
+        if (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is True
+        ):
+            owner = "anyUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is True
+            and include_templates is False
+        ):
+            owner = "anyUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "currentUserOrNoUser"
+        elif (
+            include_owned is True
+            and include_unowned is False
+            and include_templates is False
+        ):
+            owner = "currentUser"
+        elif (
+            include_owned is False
+            and include_unowned is False
+            and include_templates is True
+        ):
+            owner = "noUser"
         else:
             return []
 
-        return super()._get(params={
-            'owner': owner
-        })
+        return super()._get(params={"owner": owner})
 
     @expand_docstring(include_uid=True)
     def get(self, uid: Union[UUID, str]) -> ResultQualifier: