hydroserverpy 0.3.0__py3-none-any.whl → 0.5.0b1__py3-none-any.whl

hydroserverpy/core/schemas/data_sources.py

@@ -1,206 +0,0 @@
-import tempfile
-import io
-from pydantic import BaseModel, Field
-from typing import Optional, Literal, Union, List, TYPE_CHECKING
-from datetime import datetime
-from uuid import UUID
-from urllib.request import urlopen
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-from hydroserverpy.etl.service import HydroServerETL
-
-if TYPE_CHECKING:
-    from hydroserverpy.core.schemas.data_loaders import DataLoader
-    from hydroserverpy.core.schemas.datastreams import Datastream
-
-
-class DataSourceFields(BaseModel):
-    name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the data source.'
-    )
-    path: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The path to a local data source file.'
-    )
-    link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The link to a remote data source file.'
-    )
-    header_row: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The row number where the data begins.'
-    )
-    data_start_row: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The row number where the data begins.'
-    )
-    delimiter: Optional[str] = Field(
-        ',', strip_whitespace=True, max_length=1,
-        description='The delimiter used by the data source file.'
-    )
-    quote_char: Optional[str] = Field(
-        '"', strip_whitespace=True, max_length=1,
-        description='The quote delimiter character used by the data source file.'
-    )
-    interval: Optional[int] = Field(
-        None, gt=0, lt=9999,
-        description='The time interval at which the data source should be loaded.'
-    )
-    interval_units: Optional[Literal['minutes', 'hours', 'days', 'weeks', 'months']] = Field(
-        None,
-        description='The interval units used by the data source file.'
-    )
-    crontab: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The crontab used to schedule when the data source should be loaded.'
-    )
-    start_time: Optional[datetime] = Field(
-        None,
-        description='When the data source should begin being loaded.'
-    )
-    end_time: Optional[datetime] = Field(
-        None,
-        description='When the data source should stop being loaded.'
-    )
-    paused: Optional[bool] = Field(
-        False,
-        description='Whether loading the data source should be paused or not.'
-    )
-    timestamp_column: Union[int, str] = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The column of the data source file containing the timestamps.'
-    )
-    timestamp_format: Optional[str] = Field(
-        '%Y-%m-%dT%H:%M:%S%Z', strip_whitespace=True, max_length=255,
-        description='The format of the timestamps, using Python\'s datetime strftime codes.'
-    )
-    timestamp_offset: Optional[str] = Field(
-        '+0000', strip_whitespace=True, max_length=255,
-        description='An ISO 8601 time zone offset designator code to be applied to timestamps in the data source file.'
-    )
-    data_loader_id: UUID = Field(
-        ...,
-        description='The ID of the data loader responsible for loading this data source.'
-    )
-    data_source_thru: Optional[datetime] = Field(
-        None,
-        description='The timestamp through which the data source contains data.'
-    )
-    last_sync_successful: Optional[bool] = Field(
-        None,
-        description='Whether the last data loading attempt was successful of not.'
-    )
-    last_sync_message: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A message generated by the data loader it attempted to load data from this data source.'
-    )
-    last_synced: Optional[datetime] = Field(
-        None,
-        description='The last time the data loader attempted to load data from this data source.'
-    )
-    next_sync: Optional[datetime] = Field(
-        None,
-        description="The next time the data loader will attempt to load data from this data source."
-    )
-
-
-class DataSource(HydroServerCoreModel, DataSourceFields):
-    """
-    A model representing a data source, extending the core functionality of HydroServerCoreModel with additional
-    properties and methods.
-
-    :ivar _datastreams: A private attribute to cache the list of datastreams associated with the data source.
-    :ivar _data_loader: A private attribute to cache the data loader associated with the data source.
-    """
-
-    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
-        """
-        Initialize a DataSource instance.
-
-        :param _endpoint: The endpoint associated with the DataSource.
-        :param _uid: The unique identifier for the DataSource.
-        :type _uid: Optional[UUID]
-        :param data: Additional attributes for the DataSource.
-        """
-
-        super().__init__(_endpoint=_endpoint, _uid=_uid, **data)
-        self._datastreams = None
-        self._data_loader = None
-
-    @property
-    def datastreams(self) -> List['Datastream']:
-        """
-        Retrieve the datastreams associated with the DataSource. If not already cached, fetch the datastreams from the
-        server.
-
-        :return: A list of datastreams associated with the data source.
-        :rtype: List[Datastream]
-        """
-
-        if self._datastreams is None:
-            self._datastreams = self._endpoint.list_datastreams(uid=self.uid)
-
-        return self._datastreams
-
-    @property
-    def data_loader(self) -> 'DataLoader':
-        """
-        Retrieve the data loader associated with the data source. If not already cached, fetch the data loader from the
-        server.
-
-        :return: The data loader associated with the data source.
-        :rtype: DataLoader
-        """
-
-        if self._data_loader is None:
-            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
-
-        return self._data_loader
-
-    def refresh(self) -> None:
-        """
-        Refresh the data source with the latest data from the server and update cached datastreams and data loader if
-        they were previously loaded.
-        """
-
-        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
-        self._original_data = entity
-        self.__dict__.update(entity)
-        if self._datastreams is not None:
-            self._datastreams = self._endpoint.list_datastreams(uid=self.uid)
-        if self._data_loader is not None:
-            self._data_loader = self._endpoint._service.dataloaders.get(uid=self.data_loader_id)  # noqa
-
-    def load_observations(self) -> None:
-        """
-        Load observations data from a local file or a remote URL into HydroServer using this data source configuration.
-        """
-
-        if self.path:
-            with open(self.path, 'rb') as f:
-                with io.TextIOWrapper(f, encoding='utf-8') as data_file:
-                    hs_etl = HydroServerETL(
-                        service=getattr(self._endpoint, '_service'),
-                        data_file=data_file,
-                        data_source=self,
-                    )
-                    hs_etl.run()
-        elif self.link:
-            with tempfile.NamedTemporaryFile(mode='w+b') as temp_file:
-                with urlopen(self.link) as response:
-                    chunk_size = 1024 * 1024 * 10  # Use a 10mb chunk size.
-                    while True:
-                        chunk = response.read(chunk_size)
-                        if not chunk:
-                            break
-                        temp_file.write(chunk)
-                temp_file.seek(0)
-                with io.TextIOWrapper(temp_file, encoding='utf-8') as data_file:
-                    hs_etl = HydroServerETL(
-                        service=getattr(self._endpoint, '_service'),
-                        data_file=data_file,
-                        data_source=self,
-                    )
-                    hs_etl.run()
-        else:
-            return None

hydroserverpy/core/schemas/datastreams.py

@@ -1,299 +0,0 @@
-from pydantic import BaseModel, Field
-from pandas import DataFrame
-from typing import Optional, Literal, TYPE_CHECKING
-from uuid import UUID
-from datetime import datetime
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-
-if TYPE_CHECKING:
-    from hydroserverpy.core.schemas.things import Thing
-    from hydroserverpy.core.schemas.data_sources import DataSource
-    from hydroserverpy.core.schemas.sensors import Sensor
-    from hydroserverpy.core.schemas.units import Unit
-    from hydroserverpy.core.schemas.processing_levels import ProcessingLevel
-    from hydroserverpy.core.schemas.observed_properties import ObservedProperty
-
-
-class DatastreamFields(BaseModel):
-    name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the datastream.'
-    )
-    description: str = Field(
-        ..., strip_whitespace=True,
-        description='A description of the datastream.'
-    )
-    observation_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The type of observation recorded in this datastream'
-    )
-    sampled_medium: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The physical medium in which the observations were sampled.'
-    )
-    no_data_value: float = Field(
-        ...,
-        description='A numerical value representing no data at a given timestamp.',
-    )
-    aggregation_statistic: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The statistic calculated over the time aggregation interval of observations in this datastream.'
-    )
-    time_aggregation_interval: float = Field(
-        ...,
-        description='The time interval over which the aggregation statistic is applied to observations.',
-    )
-    status: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The current status of this datastream.'
-    )
-    result_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The type of result recorded in this datastream.'
-    )
-    value_count: Optional[int] = Field(
-        None, ge=0,
-        description='The total number of observations in this datastream.'
-    )
-    phenomenon_begin_time: Optional[datetime] = Field(
-        None,
-        description='The timestamp representing when the first phenomenon recorded in this datastream occurred.'
-    )
-    phenomenon_end_time: Optional[datetime] = Field(
-        None,
-        description='The timestamp representing when the last phenomenon recorded in this datastream occurred.'
-    )
-    result_begin_time: Optional[datetime] = Field(
-        None,
-        description='The timestamp representing when the first observation of this datastream was recorded.'
-    )
-    result_end_time: Optional[datetime] = Field(
-        None,
-        description='The timestamp representing when the last observation of this datastream was recorded.'
-    )
-    data_source_id: Optional[UUID] = Field(
-        None,
-        description='The data source for observations of this datastream.'
-    )
-    data_source_column: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The name of the column containing this datastream\'s observations in the data source file.'
-    )
-    is_visible: bool = Field(
-        True,
-        description='Whether this datastream is publicly visible.'
-    )
-    is_data_visible: bool = Field(
-        True,
-        description='Whether this observations associated with this datastream are publicly visible.'
-    )
-    thing_id: UUID = Field(
-        ...,
-        description='The site/thing from which observations of this datastream were recorded.'
-    )
-    sensor_id: UUID = Field(
-        ...,
-        description='The sensor used to record observations of this datastream.'
-    )
-    observed_property_id: UUID = Field(
-        ...,
-        description='The physical property being observed for this datastream.'
-    )
-    processing_level_id: UUID = Field(
-        ...,
-        description='The processing level applied to this datastream.'
-    )
-    unit_id: UUID = Field(
-        ...,
-        description='The unit used to record observations for this datastream.'
-    )
-    time_aggregation_interval_units: Literal['seconds', 'minutes', 'hours', 'days'] = Field(
-        ...,
-        description='The time unit for this datastream\'s time aggregation interval'
-    )
-    intended_time_spacing: Optional[float] = Field(
-        None,
-        description='The time interval at which observations should be made for this datastream.'
-    )
-    intended_time_spacing_units: Optional[Literal['seconds', 'minutes', 'hours', 'days']] = Field(
-        None,
-        description='The time unit for this datastream\'s intended time spacing interval'
-    )
-
-
-class Datastream(HydroServerCoreModel, DatastreamFields):
-    """
-    A model representing a datastream, extending the core functionality of HydroServerCoreModel with additional
-    properties and methods.
-
-    :ivar _thing: A private attribute to cache the associated thing entity.
-    :ivar _data_source: A private attribute to cache the associated data source entity.
-    :ivar _observed_property: A private attribute to cache the associated observed property entity.
-    :ivar _processing_level: A private attribute to cache the associated processing level entity.
-    :ivar _unit: A private attribute to cache the associated unit entity.
-    :ivar _sensor: A private attribute to cache the associated sensor entity.
-    """
-
-    def __init__(self, _endpoint, _uid: Optional[UUID] = None, **data):
-        """
-        Initialize a Datastream instance.
-
-        :param _endpoint: The endpoint associated with the Datastream.
-        :param _uid: The unique identifier for the Datastream.
-        :type _uid: Optional[UUID]
-        :param data: Additional attributes for the Datastream.
-        """
-
-        super().__init__(_endpoint=_endpoint, _uid=_uid, **data)
-        self._thing = None
-        self._data_source = None
-        self._observed_property = None
-        self._processing_level = None
-        self._unit = None
-        self._sensor = None
-
-    @property
-    def thing(self) -> 'Thing':
-        """
-        The thing entity associated with the datastream. If not already cached, fetch it from the server.
-
-        :return: The thing entity associated with the datastream.
-        :rtype: Thing
-        """
-
-        if self._thing is None:
-            self._thing = self._endpoint._service.things.get(uid=self.thing_id)  # noqa
-
-        return self._thing
-
-    @property
-    def data_source(self) -> 'DataSource':
-        """
-        The data source entity associated with the datastream. If not already cached, fetch it from the server.
-
-        :return: The data source entity associated with the datastream.
-        :rtype: DataSource
-        """
-
-        if self._data_source is None:
-            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
-
-        return self._data_source
-
-    @property
-    def observed_property(self) -> 'ObservedProperty':
-        """
-        Retrieve the observed property entity associated with the datastream. If not already cached, fetch it from the
-        server.
-
-        :return: The observed property entity associated with the datastream.
-        :rtype: ObservedProperty
-        """
-
-        if self._observed_property is None:
-            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
-
-        return self._observed_property
-
-    @property
-    def processing_level(self) -> 'ProcessingLevel':
-        """
-        Retrieve the processing level entity associated with the datastream. If not already cached, fetch it from the
-        server.
-
-        :return: The processing level entity associated with the datastream.
-        :rtype: ProcessingLevel
-        """
-
-        if self._processing_level is None:
-            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
-
-        return self._processing_level
-
-    @property
-    def unit(self) -> 'Unit':
-        """
-        Retrieve the unit entity associated with the datastream. If not already cached, fetch it from the server.
-
-        :return: The unit entity associated with the datastream.
-        :rtype: Unit
-        """
-
-        if self._unit is None:
-            self._unit = self._endpoint._service.units.get(uid=self.unit_id)  # noqa
-
-        return self._unit
-
-    @property
-    def sensor(self) -> 'Sensor':
-        """
-        Retrieve the sensor entity associated with the datastream. If not already cached, fetch it from the server.
-
-        :return: The sensor entity associated with the datastream.
-        :rtype: Any
-        """
-
-        if self._sensor is None:
-            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
-
-        return self._sensor
-
-    def refresh(self) -> None:
-        """
-        Refresh the datastream with the latest data from the server and update cached entities if they were previously
-        loaded.
-        """
-
-        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
-        self._original_data = entity
-        self.__dict__.update(entity)
-        if self._thing is not None:
-            self._thing = self._endpoint._service.things.get(uid=self.thing_id)  # noqa
-        if self._data_source is not None:
-            self._data_source = self._endpoint._service.datasources.get(uid=self.data_source_id)  # noqa
-        if self._observed_property is not None:
-            self._observed_property = self._endpoint._service.observedproperties.get(uid=self.observed_property_id)  # noqa
-        if self._processing_level is not None:
-            self._processing_level = self._endpoint._service.processinglevels.get(uid=self.processing_level_id)  # noqa
-        if self._unit is not None:
-            self._unit = self._endpoint._service.units.get(uid=self.unit_id)  # noqa
-        if self._sensor is not None:
-            self._sensor = self._endpoint._service.sensors.get(uid=self.sensor_id)  # noqa
-
-    def get_observations(
-            self,
-            start_time: datetime = None,
-            end_time: datetime = None,
-            page: int = 1,
-            page_size: int = 100000,
-            include_quality: bool = False,
-            fetch_all: bool = False
-    ) -> DataFrame:
-        """
-        Retrieve the observations for this datastream.
-
-        :return: A DataFrame containing the observations associated with the datastream.
-        :rtype: DataFrame
-        """
-
-        return self._endpoint.get_observations(
-            uid=self.uid, start_time=start_time, end_time=end_time, page=page, page_size=page_size,
-            include_quality=include_quality, fetch_all=fetch_all
-        )
-
-    def load_observations(
-            self,
-            observations: DataFrame,
-    ) -> None:
-        """
-        Load a DataFrame of observations to the datastream.
-
-        :param observations: A pandas DataFrame containing the observations to be uploaded.
-        :type observations: DataFrame
-        :return: None
-        """
-
-        return self._endpoint.load_observations(
-            uid=self.uid,
-            observations=observations,
-        )

hydroserverpy/core/schemas/observed_properties.py

@@ -1,35 +0,0 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-
-
-class ObservedPropertyFields(BaseModel):
-    name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the observed property.'
-    )
-    definition: str = Field(
-        ..., strip_whitespace=True,
-        description='The definition of the observed property.'
-    )
-    description: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A description of the observed property.'
-    )
-    type: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The type of the observed property.'
-    )
-    code: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='A code representing the observed property.'
-    )
-
-
-class ObservedProperty(HydroServerCoreModel, ObservedPropertyFields):
-    """
-    A model representing an observed property, extending the core functionality of HydroServerCoreModel with additional
-    fields defined in ObservedPropertyFields.
-    """
-
-    pass

hydroserverpy/core/schemas/processing_levels.py

@@ -1,27 +0,0 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-
-
-class ProcessingLevelFields(BaseModel):
-    code: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='A code representing the processing level.'
-    )
-    definition: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='The definition of the processing level.'
-    )
-    explanation: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='The explanation of the processing level.'
-    )
-
-
-class ProcessingLevel(HydroServerCoreModel, ProcessingLevelFields):
-    """
-    A model representing a processing level, extending the core functionality of HydroServerCoreModel with additional
-    fields defined in ProcessingLevelFields.
-    """
-
-    pass

hydroserverpy/core/schemas/result_qualifiers.py

@@ -1,23 +0,0 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-
-
-class ResultQualifierFields(BaseModel):
-    code: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='A code representing the result qualifier.'
-    )
-    description: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='A description of the result qualifier.'
-    )
-
-
-class ResultQualifier(HydroServerCoreModel, ResultQualifierFields):
-    """
-    A model representing an result qualifier, extending the core functionality of HydroServerCoreModel with additional
-    fields defined in ResultQualifierFields.
-    """
-
-    pass

hydroserverpy/core/schemas/sensors.py

@@ -1,53 +0,0 @@
-from pydantic import BaseModel, Field, ConfigDict
-from typing import Optional
-from hydroserverpy.core.schemas.base import HydroServerCoreModel
-
-
-class SensorFields(BaseModel):
-    name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the sensor.'
-    )
-    description: str = Field(
-        strip_whitespace=True,
-        description='A description of the sensor.'
-    )
-    encoding_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The encoding type of the sensor.'
-    )
-    manufacturer: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The manufacturer of the sensor.'
-    )
-    model: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The model of the sensor.'
-    )
-    model_link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=500,
-        description='A link to a website or file that describes the sensor model.'
-    )
-    method_type: str = Field(
-        ..., strip_whitespace=True, max_length=100,
-        description='The type of method used by this sensor to collect observations.'
-    )
-    method_link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=500,
-        description='A link to a website or file that describes the sensor method.'
-    )
-    method_code: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=50,
-        description='A code representing the sensor method.'
-    )
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class Sensor(HydroServerCoreModel, SensorFields):
-    """
-    A model representing a sensor, extending the core functionality of HydroServerCoreModel with additional
-    fields defined in SensorFields.
-    """
-
-    pass