hydroserverpy 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

hydroserverpy/core/schemas/base.py

@@ -0,0 +1,117 @@
+from pydantic import BaseModel, PrivateAttr, AliasGenerator, AliasChoices, computed_field
+from pydantic.alias_generators import to_camel
+from uuid import UUID
+from typing import Optional
+
+
+base_alias_generator = AliasGenerator(
+    serialization_alias=lambda field_name: to_camel(field_name),
+    validation_alias=lambda field_name: AliasChoices(to_camel(field_name), field_name),
+)
+
+
+class HydroServerBaseModel(BaseModel):
+    """
+    A base model for HydroServer entities that provides common attributes and functionality for HydroServer data.
+
+    :ivar _uid: A private attribute for storing the unique identifier (UUID) of the model.
+    """
+
+    _uid: Optional[UUID] = PrivateAttr()
+
+    def __init__(self, _uid: Optional[UUID] = None, **data):
+        """
+        Initialize a HydroServerBaseModel instance.
+
+        :param _uid: The unique identifier for the model.
+        :type _uid: Optional[UUID]
+        :param data: Additional attributes for the model.
+        """
+
+        super().__init__(**data)
+        self._uid = _uid
+
+    @computed_field
+    @property
+    def uid(self) -> Optional[UUID]:
+        """
+        The unique identifier (UUID) of the model.
+
+        :return: The UUID of the model.
+        :rtype: Optional[UUID]
+        """
+
+        return self._uid
+
+    class Config:
+        alias_generator = base_alias_generator
+        validate_assignment = True
+
+
+class HydroServerCoreModel(HydroServerBaseModel):
+    """
+    A core model for HydroServer entities that includes methods for data manipulation and persistence.
+
+    :ivar _original_data: A private attribute storing the original data used to initialize the model.
+    """
+
+    _original_data: Optional[dict] = PrivateAttr()
+
+    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
+        """
+        Initialize a HydroServerCoreModel instance.
+
+        :param _endpoint: The endpoint associated with the model.
+        :param _uid: The unique identifier for the model.
+        :type _uid: Optional[UUID]
+        :param data: Additional attributes for the model.
+        """
+
+        super().__init__(_uid=_uid, **data)
+        self._endpoint = _endpoint
+        self._original_data = self.dict(by_alias=False).copy()
+
+    @property
+    def _patch_data(self) -> dict:
+        """
+        Generate a dictionary of modified data that needs to be patched on the server.
+
+        :return: A dictionary of modified attributes.
+        :rtype: dict
+        """
+
+        return {
+            key: getattr(self, key) for key, value in self._original_data.items()
+            if hasattr(self, key) and getattr(self, key) != value
+        }
+
+    def refresh(self) -> None:
+        """
+        Refresh the model with the latest data from the server.
+        """
+
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        self._original_data = entity.dict(by_alias=False, exclude=['uid'])
+        self.__dict__.update(self._original_data)
+
+    def save(self) -> None:
+        """
+        Save the current state of the model to the server by updating modified attributes.
+        """
+
+        if self._patch_data:
+            entity = self._endpoint.update(uid=self.uid, **self._patch_data)
+            self._original_data = entity.dict(by_alias=False, exclude=['uid'])
+            self.__dict__.update(self._original_data)
+
+    def delete(self) -> None:
+        """
+        Delete the model from the server.
+
+        :raises AttributeError: If the model's UID is not set.
+        """
+
+        if not self._uid:
+            raise AttributeError('This resource cannot be deleted: UID is not set.')
+        self._endpoint.delete(uid=self._uid)
+        self._uid = None

hydroserverpy/core/schemas/data_loaders.py

@@ -0,0 +1,71 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List, TYPE_CHECKING
+from uuid import UUID
+from hydroserverpy.core.schemas.base import HydroServerCoreModel
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.schemas.data_sources import DataSource
+
+
+class DataLoaderFields(BaseModel):
+    name: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The name of the data loader.'
+    )
+
+
+class DataLoader(HydroServerCoreModel, DataLoaderFields):
+    """
+    A model representing a DataLoader, extending the core functionality of HydroServerCoreModel with additional
+    properties and methods.
+
+    :ivar _data_sources: A private attribute to cache the list of data sources associated with the DataLoader.
+    """
+
+    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
+        """
+        Initialize a DataLoader instance.
+
+        :param _endpoint: The endpoint associated with the data loader.
+        :type _endpoint: str
+        :param _uid: The unique identifier for the data loader.
+        :type _uid: Optional[UUID]
+        :param data: Additional attributes for the data loader.
+        """
+
+        super().__init__(_endpoint=_endpoint, _uid=_uid, **data)
+        self._data_sources = None
+
+    @property
+    def data_sources(self) -> List['DataSource']:
+        """
+        The data sources associated with the data loader. If not already cached, fetch the data sources from the server.
+
+        :return: A list of data sources associated with the data loader.
+        :rtype: List[DataSource]
+        """
+
+        if self._data_sources is None:
+            self._data_sources = self._endpoint.list_data_sources(uid=self.uid)
+
+        return self._data_sources
+
+    def refresh(self) -> None:
+        """
+        Refresh the data loader with the latest data from the server and update cached data sources.
+        """
+
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        self._original_data = entity
+        self.__dict__.update(entity)
+        if self._data_sources is not None:
+            self._data_sources = self._endpoint.list_data_sources(uid=self.uid)
+
+    def load_observations(self) -> None:
+        """
+        Load observations data from a local file or a remote URL into HydroServer using all data sources associated with
+        this data loader.
+        """
+
+        for data_source in self.data_sources:
+            data_source.load_observations()

hydroserverpy/core/schemas/data_sources.py

@@ -0,0 +1,206 @@
+import tempfile
+import io
+from pydantic import BaseModel, Field
+from typing import Optional, Literal, Union, List, TYPE_CHECKING
+from datetime import datetime
+from uuid import UUID
+from urllib.request import urlopen
+from hydroserverpy.core.schemas.base import HydroServerCoreModel
+from hydroserverpy.etl.service import HydroServerETL
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.schemas.data_loaders import DataLoader
+    from hydroserverpy.core.schemas.datastreams import Datastream
+
+
+class DataSourceFields(BaseModel):
+    name: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The name of the data source.'
+    )
+    path: Optional[str] = Field(
+        None, strip_whitespace=True, max_length=255,
+        description='The path to a local data source file.'
+    )
+    link: Optional[str] = Field(
+        None, strip_whitespace=True, max_length=255,
+        description='The link to a remote data source file.'
+    )
+    header_row: Optional[int] = Field(
+        None, gt=0, lt=9999,
+        description='The row number where the data begins.'
+    )
+    data_start_row: Optional[int] = Field(
+        None, gt=0, lt=9999,
+        description='The row number where the data begins.'
+    )
+    delimiter: Optional[str] = Field(
+        ',', strip_whitespace=True, max_length=1,
+        description='The delimiter used by the data source file.'
+    )
+    quote_char: Optional[str] = Field(
+        '"', strip_whitespace=True, max_length=1,
+        description='The quote delimiter character used by the data source file.'
+    )
+    interval: Optional[int] = Field(
+        None, gt=0, lt=9999,
+        description='The time interval at which the data source should be loaded.'
+    )
+    interval_units: Optional[Literal['minutes', 'hours', 'days', 'weeks', 'months']] = Field(
+        None,
+        description='The interval units used by the data source file.'
+    )
+    crontab: Optional[str] = Field(
+        None, strip_whitespace=True, max_length=255,
+        description='The crontab used to schedule when the data source should be loaded.'
+    )
+    start_time: Optional[datetime] = Field(
+        None,
+        description='When the data source should begin being loaded.'
+    )
+    end_time: Optional[datetime] = Field(
+        None,
+        description='When the data source should stop being loaded.'
+    )
+    paused: Optional[bool] = Field(
+        False,
+        description='Whether loading the data source should be paused or not.'
+    )
+    timestamp_column: Union[int, str] = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The column of the data source file containing the timestamps.'
+    )
+    timestamp_format: Optional[str] = Field(
+        '%Y-%m-%dT%H:%M:%S%Z', strip_whitespace=True, max_length=255,
+        description='The format of the timestamps, using Python\'s datetime strftime codes.'
+    )
+    timestamp_offset: Optional[str] = Field(
+        '+0000', strip_whitespace=True, max_length=255,
+        description='An ISO 8601 time zone offset designator code to be applied to timestamps in the data source file.'
+    )
+    data_loader_id: UUID = Field(
+        ...,
+        description='The ID of the data loader responsible for loading this data source.'
+    )
+    data_source_thru: Optional[datetime] = Field(
+        None,
+        description='The timestamp through which the data source contains data.'
+    )
+    last_sync_successful: Optional[bool] = Field(
+        None,
+        description='Whether the last data loading attempt was successful of not.'
+    )
+    last_sync_message: Optional[str] = Field(
+        None, strip_whitespace=True,
+        description='A message generated by the data loader it attempted to load data from this data source.'
+    )
+    last_synced: Optional[datetime] = Field(
+        None,
+        description='The last time the data loader attempted to load data from this data source.'
+    )
+    next_sync: Optional[datetime] = Field(
+        None,
+        description="The next time the data loader will attempt to load data from this data source."
+    )
+
+
+class DataSource(HydroServerCoreModel, DataSourceFields):
+    """
+    A model representing a data source, extending the core functionality of HydroServerCoreModel with additional
+    properties and methods.
+
+    :ivar _datastreams: A private attribute to cache the list of datastreams associated with the data source.
+    :ivar _data_loader: A private attribute to cache the data loader associated with the data source.
+    """
+
+    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
+        """
+        Initialize a DataSource instance.
+
+        :param _endpoint: The endpoint associated with the DataSource.
+        :param _uid: The unique identifier for the DataSource.
+        :type _uid: Optional[UUID]
+        :param data: Additional attributes for the DataSource.
+        """
+
+        super().__init__(_endpoint=_endpoint, _uid=_uid, **data)
+        self._datastreams = None
+        self._data_loader = None
+
+    @property
+    def datastreams(self) -> List['Datastream']:
+        """
+        Retrieve the datastreams associated with the DataSource. If not already cached, fetch the datastreams from the
+        server.
+
+        :return: A list of datastreams associated with the data source.
+        :rtype: List[Datastream]
+        """
+
+        if self._datastreams is None:
+            self._datastreams = self._endpoint.list_datastreams(uid=self.uid)
+
+        return self._datastreams
+
+    @property
+    def data_loader(self) -> 'DataLoader':
+        """
+        Retrieve the data loader associated with the data source. If not already cached, fetch the data loader from the
+        server.
+
+        :return: The data loader associated with the data source.
+        :rtype: DataLoader
+        """
+
+        if self._data_loader is None:
+            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
+
+        return self._data_loader
+
+    def refresh(self) -> None:
+        """
+        Refresh the data source with the latest data from the server and update cached datastreams and data loader if
+        they were previously loaded.
+        """
+
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        self._original_data = entity
+        self.__dict__.update(entity)
+        if self._datastreams is not None:
+            self._datastreams = self._endpoint.list_datastreams(uid=self.uid)
+        if self._data_loader is not None:
+            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
+
+    def load_observations(self) -> None:
+        """
+        Load observations data from a local file or a remote URL into HydroServer using this data source configuration.
+        """
+
+        if self.path:
+            with open(self.path, 'rb') as f:
+                with io.TextIOWrapper(f, encoding='utf-8') as data_file:
+                    hs_etl = HydroServerETL(
+                        service=getattr(self._endpoint, '_service'),
+                        data_file=data_file,
+                        data_source=self,
+                    )
+                    hs_etl.run()
+        elif self.link:
+            with tempfile.NamedTemporaryFile(mode='w+b') as temp_file:
+                with urlopen(self.link) as response:
+                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        temp_file.write(chunk)
+                temp_file.seek(0)
+                with io.TextIOWrapper(temp_file, encoding='utf-8') as data_file:
+                    hs_etl = HydroServerETL(
+                        service=getattr(self._endpoint, '_service'),
+                        data_file=data_file,
+                        data_source=self,
+                    )
+                    hs_etl.run()
+        else:
+            return None

hydroserverpy/core/schemas/datastreams.py

@@ -0,0 +1,299 @@
+from pydantic import BaseModel, Field
+from pandas import DataFrame
+from typing import Optional, Literal, TYPE_CHECKING
+from uuid import UUID
+from datetime import datetime
+from hydroserverpy.core.schemas.base import HydroServerCoreModel
+
+if TYPE_CHECKING:
+    from hydroserverpy.core.schemas.things import Thing
+    from hydroserverpy.core.schemas.data_sources import DataSource
+    from hydroserverpy.core.schemas.sensors import Sensor
+    from hydroserverpy.core.schemas.units import Unit
+    from hydroserverpy.core.schemas.processing_levels import ProcessingLevel
+    from hydroserverpy.core.schemas.observed_properties import ObservedProperty
+
+
+class DatastreamFields(BaseModel):
+    name: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The name of the datastream.'
+    )
+    description: str = Field(
+        ..., strip_whitespace=True,
+        description='A description of the datastream.'
+    )
+    observation_type: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The type of observation recorded in this datastream'
+    )
+    sampled_medium: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The physical medium in which the observations were sampled.'
+    )
+    no_data_value: float = Field(
+        ...,
+        description='A numerical value representing no data at a given timestamp.',
+    )
+    aggregation_statistic: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The statistic calculated over the time aggregation interval of observations in this datastream.'
+    )
+    time_aggregation_interval: float = Field(
+        ...,
+        description='The time interval over which the aggregation statistic is applied to observations.',
+    )
+    status: Optional[str] = Field(
+        None, strip_whitespace=True, max_length=255,
+        description='The current status of this datastream.'
+    )
+    result_type: str = Field(
+        ..., strip_whitespace=True, max_length=255,
+        description='The type of result recorded in this datastream.'
+    )
+    value_count: Optional[int] = Field(
+        None, ge=0,
+        description='The total number of observations in this datastream.'
+    )
+    phenomenon_begin_time: Optional[datetime] = Field(
+        None,
+        description='The timestamp representing when the first phenomenon recorded in this datastream occurred.'
+    )
+    phenomenon_end_time: Optional[datetime] = Field(
+        None,
+        description='The timestamp representing when the last phenomenon recorded in this datastream occurred.'
+    )
+    result_begin_time: Optional[datetime] = Field(
+        None,
+        description='The timestamp representing when the first observation of this datastream was recorded.'
+    )
+    result_end_time: Optional[datetime] = Field(
+        None,
+        description='The timestamp representing when the last observation of this datastream was recorded.'
+    )
+    data_source_id: Optional[UUID] = Field(
+        None,
+        description='The data source for observations of this datastream.'
+    )
+    data_source_column: Optional[str] = Field(
+        None, strip_whitespace=True, max_length=255,
+        description='The name of the column containing this datastream\'s observations in the data source file.'
+    )
+    is_visible: bool = Field(
+        True,
+        description='Whether this datastream is publicly visible.'
+    )
+    is_data_visible: bool = Field(
+        True,
+        description='Whether this observations associated with this datastream are publicly visible.'
+    )
+    thing_id: UUID = Field(
+        ...,
+        description='The site/thing from which observations of this datastream were recorded.'
+    )
+    sensor_id: UUID = Field(
+        ...,
+        description='The sensor used to record observations of this datastream.'
+    )
+    observed_property_id: UUID = Field(
+        ...,
+        description='The physical property being observed for this datastream.'
+    )
+    processing_level_id: UUID = Field(
+        ...,
+        description='The processing level applied to this datastream.'
+    )
+    unit_id: UUID = Field(
+        ...,
+        description='The unit used to record observations for this datastream.'
+    )
+    time_aggregation_interval_units: Literal['seconds', 'minutes', 'hours', 'days'] = Field(
+        ...,
+        description='The time unit for this datastream\'s time aggregation interval'
+    )
+    intended_time_spacing: Optional[float] = Field(
+        None,
+        description='The time interval at which observations should be made for this datastream.'
+    )
+    intended_time_spacing_units: Optional[Literal['seconds', 'minutes', 'hours', 'days']] = Field(
+        None,
+        description='The time unit for this datastream\'s intended time spacing interval'
+    )
+
+
+class Datastream(HydroServerCoreModel, DatastreamFields):
+    """
+    A model representing a datastream, extending the core functionality of HydroServerCoreModel with additional
+    properties and methods.
+
+    :ivar _thing: A private attribute to cache the associated thing entity.
+    :ivar _data_source: A private attribute to cache the associated data source entity.
+    :ivar _observed_property: A private attribute to cache the associated observed property entity.
+    :ivar _processing_level: A private attribute to cache the associated processing level entity.
+    :ivar _unit: A private attribute to cache the associated unit entity.
+    :ivar _sensor: A private attribute to cache the associated sensor entity.
+    """
+
+    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
+        """
+        Initialize a Datastream instance.
+
+        :param _endpoint: The endpoint associated with the Datastream.
+        :param _uid: The unique identifier for the Datastream.
+        :type _uid: Optional[UUID]
+        :param data: Additional attributes for the Datastream.
+        """
+
+        super().__init__(_endpoint=_endpoint, _uid=_uid, **data)
+        self._thing = None
+        self._data_source = None
+        self._observed_property = None
+        self._processing_level = None
+        self._unit = None
+        self._sensor = None
+
+    @property
+    def thing(self) -> 'Thing':
+        """
+        The thing entity associated with the datastream. If not already cached, fetch it from the server.
+
+        :return: The thing entity associated with the datastream.
+        :rtype: Thing
+        """
+
+        if self._thing is None:
+            self._thing = self._endpoint._service.things.get(uid=self.thing_id)  # noqa
+
+        return self._thing
+
+    @property
+    def data_source(self) -> 'DataSource':
+        """
+        The data source entity associated with the datastream. If not already cached, fetch it from the server.
+
+        :return: The data source entity associated with the datastream.
+        :rtype: DataSource
+        """
+
+        if self._data_source is None:
+            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
+
+        return self._data_source
+
+    @property
+    def observed_property(self) -> 'ObservedProperty':
+        """
+        Retrieve the observed property entity associated with the datastream. If not already cached, fetch it from the
+        server.
+
+        :return: The observed property entity associated with the datastream.
+        :rtype: ObservedProperty
+        """
+
+        if self._observed_property is None:
+            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
+
+        return self._observed_property
+
+    @property
+    def processing_level(self) -> 'ProcessingLevel':
+        """
+        Retrieve the processing level entity associated with the datastream. If not already cached, fetch it from the
+        server.
+
+        :return: The processing level entity associated with the datastream.
+        :rtype: ProcessingLevel
+        """
+
+        if self._processing_level is None:
+            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
+
+        return self._processing_level
+
+    @property
+    def unit(self) -> 'Unit':
+        """
+        Retrieve the unit entity associated with the datastream. If not already cached, fetch it from the server.
+
+        :return: The unit entity associated with the datastream.
+        :rtype: Unit
+        """
+
+        if self._unit is None:
+            self._unit = self._endpoint._service.units.get(uid=self.unit_id)  # noqa
+
+        return self._unit
+
+    @property
+    def sensor(self) -> 'Sensor':
+        """
+        Retrieve the sensor entity associated with the datastream. If not already cached, fetch it from the server.
+
+        :return: The sensor entity associated with the datastream.
+        :rtype: Any
+        """
+
+        if self._sensor is None:
+            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
+
+        return self._sensor
+
+    def refresh(self) -> None:
+        """
+        Refresh the datastream with the latest data from the server and update cached entities if they were previously
+        loaded.
+        """
+
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        self._original_data = entity
+        self.__dict__.update(entity)
+        if self._thing is not None:
+            self._thing = self._endpoint._service.things.get(uid=self.thing_id)  # noqa
+        if self._data_source is not None:
+            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
+        if self._observed_property is not None:
+            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
+        if self._processing_level is not None:
+            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
+        if self._unit is not None:
+            self._unit = self._endpoint._service.units.get(uid=self.unit_id)  # noqa
+        if self._sensor is not None:
+            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
+
+    def get_observations(
+            self,
+            start_time: datetime = None,
+            end_time: datetime = None,
+            page: int = 1,
+            page_size: int = 100000,
+            include_quality: bool = False,
+            fetch_all: bool = False
+    ) -> DataFrame:
+        """
+        Retrieve the observations for this datastream.
+
+        :return: A DataFrame containing the observations associated with the datastream.
+        :rtype: DataFrame
+        """
+
+        return self._endpoint.get_observations(
+            uid=self.uid, start_time=start_time, end_time=end_time, page=page, page_size=page_size,
+            include_quality=include_quality, fetch_all=fetch_all
+        )
+
+    def load_observations(
+            self,
+            observations: DataFrame,
+    ) -> None:
+        """
+        Load a DataFrame of observations to the datastream.
+
+        :param observations: A pandas DataFrame containing the observations to be uploaded.
+        :type observations: DataFrame
+        :return: None
+        """
+
+        return self._endpoint.load_observations(
+            uid=self.uid,
+            observations=observations,
+        )