hydroserverpy 0.2.5__tar.gz → 0.3.0__tar.gz

{hydroserverpy-0.2.5/src/hydroserverpy.egg-info → hydroserverpy-0.3.0}/PKG-INFO

@@ -1,14 +1,17 @@
 Metadata-Version: 2.1
 Name: hydroserverpy
-Version: 0.2.5
+Version: 0.3.0
+Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2
-Requires-Dist: pydantic<2.0.0,>=1.6
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pandas>=2.2
+Requires-Dist: numpy>=2.0
 Requires-Dist: pyyaml>=5
 Requires-Dist: simplejson>=3
 Requires-Dist: crontab>=1
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: croniter>=2.0.1
-Requires-Dist: frost-sta-client>=1.1.44
+Requires-Dist: country-list>=1.1.0
 Provides-Extra: docs
 Requires-Dist: sphinx_autodoc_typehints; extra == "docs"

hydroserverpy-0.3.0/README.md

@@ -0,0 +1,43 @@
+# HydroServer Python Client
+
+The hydroserverpy Python package provides an interface for managing HydroServer data and metadata, loading observations, and performing data quality control. This guide will go over how to install the package and connect to a HydroServer instance. Full hydroserverpy documentation can be found [here](https://hydroserver2.github.io/hydroserverpy).
+
+## Installation
+
+You can install the package via pip:
+
+```bash
+pip install hydroserverpy
+```
+
+## Connecting to HydroServer
+
+To connect to HydroServer, you need to initialize the client with the instance of HydroServer you're using and your user credentials if you want to access and modify your own data. If you don't provide authentication credentials you can read public data, but you will not be able to create or modify any data.
+
+### Example: Anonymous User
+
+```python
+from hydroserverpy import HydroServer
+
+# Initialize HydroServer connection.
+hs_api = HydroServer(
+    host='https://playground.hydroserver.org'
+)
+```
+
+### Example: Basic Authentication
+
+```python
+from hydroserverpy import HydroServer
+
+# Initialize HydroServer connection with credentials.
+hs_api = HydroServer(
+    host='https://playground.hydroserver.org',
+    username='user@example.com',
+    password='******'
+)
+```
+
+## Funding and Acknowledgements
+
+Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama (NA22NWS4320003).

{hydroserverpy-0.2.5 → hydroserverpy-0.3.0}/setup.cfg

@@ -1,21 +1,24 @@
 [metadata]
 name = hydroserverpy
-version = 0.2.5
+version = 0.3.0
 
 [options]
 package_dir = 
 	=src
 packages = find:
 zip_safe = True
+python_requires = >=3.9, <4
 install_requires = 
 	requests >= 2
-	pydantic >= 1.6, < 2.0.0
+	pydantic >= 2.6
+	pandas >= 2.2
+	numpy >= 2.0
 	pyyaml >= 5
 	simplejson >= 3
 	crontab >= 1
 	python-dateutil >= 2.8.2
 	croniter >= 2.0.1
-	frost-sta-client >= 1.1.44
+	country-list >= 1.1.0
 
 [options.extras_require]
 docs = 

hydroserverpy-0.3.0/src/hydroserverpy/__init__.py

@@ -0,0 +1,10 @@
+from .core.service import HydroServer
+from .quality.service import HydroServerQualityControl
+from .etl.service import HydroServerETL
+
+
+__all__ = [
+    "HydroServer",
+    "HydroServerQualityControl",
+    "HydroServerETL",
+]

hydroserverpy-0.3.0/src/hydroserverpy/core/endpoints/__init__.py

@@ -0,0 +1,9 @@
+from .data_loaders import DataLoaderEndpoint
+from .data_sources import DataSourceEndpoint
+from .datastreams import DatastreamEndpoint
+from .observed_properties import ObservedPropertyEndpoint
+from .processing_levels import ProcessingLevelEndpoint
+from .result_qualifiers import ResultQualifierEndpoint
+from .sensors import SensorEndpoint
+from .things import ThingEndpoint
+from .units import UnitEndpoint

hydroserverpy-0.3.0/src/hydroserverpy/core/endpoints/base.py

@@ -0,0 +1,133 @@
+import json
+from uuid import UUID
+from typing import TYPE_CHECKING, Type, Union, List, TypeVar, Optional
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.core.schemas.base import HydroServerCoreModel
+
+    HydroServerModelType = TypeVar('HydroServerModelType', bound=HydroServerCoreModel)
+
+
+def expand_docstring(model: Optional[Type['HydroServerCoreModel']] = None, include_uid: bool = False):
+    def decorator(func):
+        docstring = func.__doc__
+        if model is not None or include_uid is True:
+            docstring += '\n'
+        if include_uid is True:
+            docstring += f':param uid: The entity ID.\n'
+            docstring += f':type uid: Union[UUID, str]\n'
+        if model is not None:
+            for field_name, field in model.model_fields.items():
+                docstring += f':param {field_name}: {field.description}\n'
+                docstring += f':type {field_name}: {getattr(field.annotation, "__name__", field.annotation)}\n'
+        func.__doc__ = docstring
+        return func
+    return decorator
+
+
+class HydroServerEndpoint:
+    """
+    A base class for interacting with specific API endpoints within a HydroServer service.
+
+    :ivar _model: The model class associated with this endpoint.
+    :ivar _api_route: The base route of the API.
+    :ivar _endpoint_route: The specific route of the endpoint.
+    """
+
+    _model: Type['HydroServerCoreModel']
+    _api_route: str
+    _endpoint_route: str
+
+    def __init__(self, service: 'HydroServer') -> None:
+        """
+        Initialize the HydroServerEndpoint.
+
+        :param service: The HydroServer service instance to use for requests.
+        :type service: HydroServer
+        """
+
+        self._service = service
+
+    def _get(self, uid: Optional[Union[UUID, str]] = None, params: dict = None) -> Union[
+        List['HydroServerModelType'], 'HydroServerModelType'
+    ]:
+        """
+        Fetch an entity collection or single entity from a HydroServer endpoint.
+
+        :param uid: The unique identifier of the entity to retrieve.
+        :type uid: Optional[Union[UUID, str]]
+        :returns: A model instance representing the entity.
+        :rtype: HydroServerCoreModel
+        """
+
+        if params is None:
+            params = {}
+
+        path = f'{self._api_route}/data/{self._endpoint_route}{"/" + str(uid) if uid else ""}'
+        response = getattr(self._service, '_request')('get', path, params=params)
+
+        if uid:
+            entity = json.loads(response.content)
+            result = self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+        else:
+            result = [
+                self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+                for entity in json.loads(response.content)
+            ]
+
+        return result
+
+    def _post(self, **kwargs) -> 'HydroServerModelType':
+        """
+        Create a new entity using the endpoint.
+
+        :param kwargs: The attributes to set on the new entity.
+        :returns: A model instance representing the newly created entity.
+        :rtype: HydroServerModelType
+        """
+
+        response = getattr(self._service, '_request')(
+            'post', f'{self._api_route}/data/{self._endpoint_route}',
+            headers={'Content-type': 'application/json'},
+            data=self._model(_endpoint=self, **kwargs).json(exclude_unset=True, by_alias=True),
+        )
+        entity = json.loads(response.content)
+
+        return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+
+    def _patch(self, uid: Union[UUID, str], **kwargs) -> 'HydroServerModelType':
+        """
+        Update an existing entity in the endpoint.
+
+        :param uid: The unique identifier of the entity to update.
+        :type uid: Union[UUID, str]
+        :param kwargs: The attributes to update on the entity.
+        :returns: A model instance representing the updated entity.
+        :rtype: HydroServerModelType
+        """
+
+        response = getattr(self._service, '_request')(
+            'patch', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
+            headers={'Content-type': 'application/json'},
+            data=json.dumps({
+                self._model.model_fields[key].serialization_alias: value
+                for key, value in kwargs.items()
+            }, default=str)
+        )
+        entity = json.loads(response.content)
+
+        return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
+
+    def _delete(self, uid: Union[UUID, str]) -> None:
+        """
+        Delete an entity from the endpoint by its unique identifier.
+
+        :param uid: The unique identifier of the entity to delete.
+        :type uid: Union[UUID, str]
+        :returns: None
+        """
+
+        getattr(self._service, '_request')(
+            'delete', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
+        )

hydroserverpy-0.3.0/src/hydroserverpy/core/endpoints/data_loaders.py

@@ -0,0 +1,92 @@
+import json
+from typing import Union, List, TYPE_CHECKING
+from uuid import UUID
+from hydroserverpy.core.endpoints.base import HydroServerEndpoint, expand_docstring
+from hydroserverpy.core.endpoints.data_sources import DataSourceEndpoint
+from hydroserverpy.core.schemas import DataLoader, DataSource
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.service import HydroServer
+
+
+class DataLoaderEndpoint(HydroServerEndpoint):
+    """
+    An endpoint for interacting with DataLoader entities in the HydroServer service.
+
+    :ivar _model: The model class associated with this endpoint, set to `DataLoader`.
+    :ivar _api_route: The base route of the API, derived from the service.
+    :ivar _endpoint_route: The specific route of the endpoint, set to `'data-loaders'`.
+    """
+
+    def __init__(self, service: 'HydroServer') -> None:
+        """
+        Initialize the DataLoaderEndpoint.
+
+        :param service: The HydroServer service instance to use for requests.
+        :type service: HydroServer
+        """
+
+        super().__init__(service)
+        self._model = DataLoader
+        self._api_route = self._service.api_route
+        self._endpoint_route = 'data-loaders'
+
+    def list(self) -> List[DataLoader]:
+        """
+        Retrieve a collection of data loaders owned by the logged-in user.
+        """
+
+        return super()._get()
+
+    @expand_docstring(include_uid=True)
+    def get(self, uid: Union[UUID, str]) -> DataLoader:
+        """
+        Retrieve a data loader owned by the logged-in user.
+        """
+
+        return super()._get(uid)
+
+    @expand_docstring(model=DataLoader)
+    def create(self, **kwargs) -> DataLoader:
+        """
+        Create a new data loader in HydroServer.
+        """
+
+        return super()._post(**kwargs)
+
+    @expand_docstring(model=DataLoader, include_uid=True)
+    def update(self, uid: Union[UUID, str], **kwargs) -> DataLoader:
+        """
+        Update an existing data loader in HydroServer.
+        """
+
+        return super()._patch(uid=uid, **kwargs)
+
+    @expand_docstring(include_uid=True)
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """
+        Delete an existing data loader in HydroServer.
+        """
+
+        super()._delete(uid=uid)
+
+    def list_data_sources(self, uid: Union[UUID, str]) -> List[DataSource]:
+        """
+        Retrieve a list of data source entities associated with a specific data loader.
+
+        :param uid: The unique identifier of the data loader.
+        :type uid: Union[UUID, str]
+        :returns: A list of data sour instances associated with the data loader.
+        :rtype: List[DataSource]
+        """
+
+        response = getattr(self._service, '_request')(
+            'get', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}/data-sources'
+        )
+
+        endpoint = DataSourceEndpoint(self._service)
+
+        return [
+            DataSource(_endpoint=endpoint, _uid=UUID(str(entity.pop('id'))), **entity)
+            for entity in json.loads(response.content)
+        ]

hydroserverpy-0.3.0/src/hydroserverpy/core/endpoints/data_sources.py

@@ -0,0 +1,92 @@
+import json
+from typing import Union, List, TYPE_CHECKING
+from uuid import UUID
+from hydroserverpy.core.endpoints.base import HydroServerEndpoint, expand_docstring
+from hydroserverpy.core.endpoints.datastreams import DatastreamEndpoint
+from hydroserverpy.core.schemas import DataSource, Datastream
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.service import HydroServer
+
+
+class DataSourceEndpoint(HydroServerEndpoint):
+    """
+    An endpoint for interacting with data source entities in the HydroServer service.
+
+    :ivar _model: The model class associated with this endpoint, set to `DataSource`.
+    :ivar _api_route: The base route of the API, derived from the service.
+    :ivar _endpoint_route: The specific route of the endpoint, set to `'data-sources'`.
+    """
+
+    def __init__(self, service: 'HydroServer') -> None:
+        """
+        Initialize the DataSourceEndpoint.
+
+        :param service: The HydroServer service instance to use for requests.
+        :type service: HydroServer
+        """
+
+        super().__init__(service)
+        self._model = DataSource
+        self._api_route = self._service.api_route
+        self._endpoint_route = 'data-sources'
+
+    def list(self) -> List[DataSource]:
+        """
+        Retrieve a collection of data sources owned by the logged-in user.
+        """
+
+        return super()._get()
+
+    @expand_docstring(include_uid=True)
+    def get(self, uid: Union[UUID, str]) -> DataSource:
+        """
+        Retrieve a data source owned by the logged-in user.
+        """
+
+        return super()._get(uid)
+
+    @expand_docstring(model=DataSource)
+    def create(self, **kwargs) -> DataSource:
+        """
+        Create a new data source in HydroServer.
+        """
+
+        return super()._post(**kwargs)
+
+    @expand_docstring(model=DataSource, include_uid=True)
+    def update(self, uid: Union[UUID, str], **kwargs) -> DataSource:
+        """
+        Update an existing data source in HydroServer.
+        """
+
+        return super()._patch(uid=uid, **kwargs)
+
+    @expand_docstring(include_uid=True)
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """
+        Delete an existing data source in HydroServer.
+        """
+
+        super()._delete(uid=uid)
+
+    def list_datastreams(self, uid: Union[UUID, str]) -> List[Datastream]:
+        """
+        Retrieve a list of datastream entities associated with a specific data source.
+
+        :param uid: The unique identifier of the data source.
+        :type uid: Union[UUID, str]
+        :returns: A list of datastream instances associated with the data source.
+        :rtype: List[Datastream]
+        """
+
+        response = getattr(self._service, '_request')(
+            'get', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}/datastreams'
+        )
+
+        endpoint = DatastreamEndpoint(self._service)
+
+        return [
+            Datastream(_endpoint=endpoint, _uid=UUID(str(entity.pop('id'))), **entity)
+            for entity in json.loads(response.content)
+        ]

hydroserverpy-0.3.0/src/hydroserverpy/core/endpoints/datastreams.py

@@ -0,0 +1,188 @@
+import json
+import pandas as pd
+from typing import List, Union, TYPE_CHECKING
+from uuid import UUID
+from datetime import datetime
+from hydroserverpy.core.endpoints.base import HydroServerEndpoint, expand_docstring
+from hydroserverpy.core.schemas import Datastream
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.service import HydroServer
+
+
+class DatastreamEndpoint(HydroServerEndpoint):
+    """
+    An endpoint for interacting with datastream entities in the HydroServer service.
+
+    :ivar _model: The model class associated with this endpoint, set to `Datastream`.
+    :ivar _api_route: The base route of the API, derived from the service.
+    :ivar _endpoint_route: The specific route of the endpoint, set to `'datastreams'`.
+    """
+
+    def __init__(self, service) -> None:
+        """
+        Initialize the DatastreamEndpoint.
+
+        :param service: The HydroServer service instance to use for requests.
+        :type service: HydroServer
+        """
+
+        super().__init__(service)
+        self._model = Datastream
+        self._api_route = self._service.api_route
+        self._endpoint_route = 'datastreams'
+
+    def list(self, owned_only: bool = False, primary_owned_only: bool = False) -> List[Datastream]:
+        """
+        Retrieve a collection of datastreams owned by the logged-in user.
+
+        :param owned_only: Only list datastreams owned by the logged-in user.
+        :param primary_owned_only: Only list datastreams primary owned by the logged-in user.
+        """
+
+        return super()._get(params={
+            'owned_only': owned_only,
+            'primary_owned_only': primary_owned_only,
+        })
+
+    @expand_docstring(include_uid=True)
+    def get(self, uid: Union[UUID, str]) -> Datastream:
+        """
+        Retrieve a datastream owned by the logged-in user.
+        """
+
+        return super()._get(uid)
+
+    @expand_docstring(model=Datastream)
+    def create(self, **kwargs) -> Datastream:
+        """
+        Create a new datastream in HydroServer.
+        """
+
+        return super()._post(**kwargs)
+
+    @expand_docstring(model=Datastream, include_uid=True)
+    def update(self, uid: Union[UUID, str], **kwargs) -> Datastream:
+        """
+        Update an existing datastream in HydroServer.
+        """
+
+        return super()._patch(uid=uid, **kwargs)
+
+    @expand_docstring(include_uid=True)
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """
+        Delete an existing datastream in HydroServer.
+        """
+
+        super()._delete(uid=uid)
+
+    def get_observations(
+            self,
+            uid: Union[UUID, str],
+            start_time: datetime = None,
+            end_time: datetime = None,
+            page: int = 1,
+            page_size: int = 100000,
+            include_quality: bool = False,
+            fetch_all: bool = False
+    ) -> pd.DataFrame:
+        """
+        Retrieve observations from a specific datastream.
+
+        :param uid: The unique identifier of the datastream.
+        :type uid: Union[UUID, str]
+        :param start_time: The start time for filtering observations.
+        :type start_time: datetime, optional
+        :param end_time: The end time for filtering observations.
+        :type end_time: datetime, optional
+        :param page: The page number to retrieve (used for pagination).
+        :type page: int, optional
+        :param page_size: The number of observations per page.
+        :type page_size: int, optional
+        :param include_quality: Whether to include quality information with each observation.
+        :type include_quality: bool, optional
+        :param fetch_all: Whether to fetch all observations (ignoring pagination).
+        :type fetch_all: bool, optional
+        :returns: A DataFrame containing the retrieved observations.
+        :rtype: pd.DataFrame
+        """
+
+        filters = []
+        if start_time:
+            filters.append(f'phenomenonTime ge {start_time.strftime("%Y-%m-%dT%H:%M:%S%z")}')
+        if end_time:
+            filters.append(f'phenomenonTime le {end_time.strftime("%Y-%m-%dT%H:%M:%S%z")}')
+
+        if fetch_all:
+            page = 1
+
+        observations = []
+
+        while True:
+            response = getattr(self._service, '_request')(
+                'get', f'{self._api_route}/sensorthings/v1.1/Datastreams(\'{str(uid)}\')/Observations',
+                params={
+                    '$resultFormat': 'dataArray',
+                    '$select': f'phenomenonTime,result{",resultQuality" if include_quality else ""}',
+                    '$count': True,
+                    '$top': page_size,
+                    '$skip': (page - 1) * page_size,
+                    '$filter': ' and '.join(filters) if filters else None
+                }
+            )
+            response_content = json.loads(response.content)
+            data_array = response_content['value'][0]['dataArray'] if response_content['value'] else []
+            observations.extend([[
+                obs[0], obs[1],
+                obs[2]['qualityCode'] if obs[2]['qualityCode'] else None,
+                obs[2]['resultQualifiers'] if obs[2]['resultQualifiers'] else None
+            ] if include_quality else [obs[0], obs[1]] for obs in data_array])
+            if not fetch_all or len(data_array) < page_size:
+                break
+            page += 1
+
+        columns = ['timestamp', 'value']
+        if include_quality:
+            columns.extend(['quality_code', 'result_quality'])
+
+        data_frame = pd.DataFrame(observations, columns=columns)
+        data_frame['timestamp'] = pd.to_datetime(data_frame['timestamp'])
+
+        return data_frame
+
+    def load_observations(
+            self,
+            uid: Union[UUID, str],
+            observations: pd.DataFrame,
+    ) -> None:
+        """
+        Load observations to a specific datastream.
+
+        :param uid: The unique identifier of the datastream.
+        :type uid: Union[UUID, str]
+        :param observations: A DataFrame containing the observations to upload.
+        :type observations: pd.DataFrame
+        :returns: None
+        """
+
+        data_array = [
+            [
+                row['timestamp'].strftime('%Y-%m-%dT%H:%M:%S%z'),
+                row['value'],
+                {
+                    'qualityCode': row.get('quality_code', None),
+                    'resultQualifiers': row.get('result_qualifiers', []),
+                } if 'quality_code' in row or 'result_qualifiers' in row else {}
+            ] for _, row in observations.iterrows()
+        ]
+
+        getattr(self._service, '_request')(
+            'post', f'{self._api_route}/sensorthings/v1.1/CreateObservations',
+            headers={'Content-type': 'application/json'},
+            data=json.dumps([{
+                'Datastream': {'@iot.id': str(uid)},
+                'components': ['phenomenonTime', 'result', 'resultQuality'],
+                'dataArray': data_array
+            }]),
+        )