hydroserverpy 1.2.1__py3-none-any.whl → 1.3.0b2__py3-none-any.whl

hydroserverpy/api/models/sta/datastream.py

@@ -1,9 +1,11 @@
-from typing import Union, Optional, Literal, TYPE_CHECKING
-from pydantic import BaseModel, Field, AliasChoices, AliasPath, field_validator
-from pandas import DataFrame
+import uuid
+import pandas as pd
+from typing import List, Union, Optional, Literal, ClassVar, TYPE_CHECKING
+from pydantic import Field
 from uuid import UUID
 from datetime import datetime
-from ..base import HydroServerModel
+from hydroserverpy.api.utils import normalize_uuid
+from ..base import HydroServerBaseModel
 
 if TYPE_CHECKING:
     from hydroserverpy import HydroServer
@@ -14,132 +16,51 @@ if TYPE_CHECKING:
         ObservedProperty,
         Unit,
         ProcessingLevel,
+        DataSource,
+        DataArchive
     )
 
 
-class DatastreamFields(BaseModel):
+class Datastream(HydroServerBaseModel):
     name: str = Field(..., max_length=255)
     description: str
     observation_type: str = Field(..., max_length=255)
-    sampled_medium: str = Field(
-        ...,
-        max_length=255,
-        validation_alias=AliasChoices(
-            "sampledMedium", AliasPath("properties", "sampledMedium")
-        ),
-    )
-    no_data_value: float = Field(
-        ...,
-        validation_alias=AliasChoices(
-            "noDataValue", AliasPath("properties", "noDataValue")
-        ),
-    )
-    aggregation_statistic: str = Field(
-        ...,
-        max_length=255,
-        validation_alias=AliasChoices(
-            "aggregationStatistic", AliasPath("properties", "aggregationStatistic")
-        ),
-    )
-    time_aggregation_interval: float = Field(
-        ...,
-        validation_alias=AliasChoices(
-            "timeAggregationInterval",
-            AliasPath("properties", "timeAggregationInterval"),
-        ),
-    )
-    status: Optional[str] = Field(
-        None,
-        max_length=255,
-        validation_alias=AliasChoices("status", AliasPath("properties", "status")),
-    )
-    result_type: str = Field(
-        ...,
-        max_length=255,
-        validation_alias=AliasChoices(
-            "resultType", AliasPath("properties", "resultType")
-        ),
-    )
-    value_count: Optional[int] = Field(
-        None,
-        ge=0,
-        validation_alias=AliasChoices(
-            "valueCount", AliasPath("properties", "valueCount")
-        ),
-    )
-    phenomenon_begin_time: Optional[datetime] = Field(
-        None, validation_alias=AliasChoices("phenomenonBeginTime", "phenomenonTime")
-    )
-    phenomenon_end_time: Optional[datetime] = Field(
-        None, validation_alias=AliasChoices("phenomenonEndTime", "phenomenonTime")
-    )
-    result_begin_time: Optional[datetime] = Field(
-        None, validation_alias=AliasChoices("resultBeginTime", "resultTime")
-    )
-    result_end_time: Optional[datetime] = Field(
-        None, validation_alias=AliasChoices("resultEndTime", "resultTime")
-    )
-    is_private: bool = Field(
-        False,
-        validation_alias=AliasChoices(
-            "isPrivate", AliasPath("properties", "isPrivate")
-        ),
-    )
-    is_visible: bool = Field(
-        True,
-        validation_alias=AliasChoices(
-            "isVisible", AliasPath("properties", "isVisible")
-        ),
-    )
-    time_aggregation_interval_unit: Literal["seconds", "minutes", "hours", "days"] = (
-        Field(
-            ...,
-            validation_alias=AliasChoices(
-                "timeAggregationIntervalUnit",
-                AliasPath("properties", "timeAggregationIntervalUnitOfMeasurement"),
-            ),
-        )
-    )
-    intended_time_spacing: Optional[float] = Field(
-        None,
-        validation_alias=AliasChoices(
-            "intendedTimeSpacing", AliasPath("properties", "intendedTimeSpacing")
-        ),
-    )
+    sampled_medium: str = Field(..., max_length=255)
+    no_data_value: float
+    aggregation_statistic: str = Field(..., max_length=255)
+    time_aggregation_interval: float
+    status: Optional[str] = Field(None, max_length=255)
+    result_type: str = Field(..., max_length=255)
+    value_count: Optional[int] = Field(None, ge=0)
+    phenomenon_begin_time: Optional[datetime] = None
+    phenomenon_end_time: Optional[datetime] = None
+    result_begin_time: Optional[datetime] = None
+    result_end_time: Optional[datetime] = None
+    is_private: bool = False
+    is_visible: bool = True
+    time_aggregation_interval_unit: Literal["seconds", "minutes", "hours", "days"]
+    intended_time_spacing: Optional[float] = None
     intended_time_spacing_unit: Optional[
         Literal["seconds", "minutes", "hours", "days"]
-    ] = Field(
-        None,
-        validation_alias=AliasChoices(
-            "intendedTimeSpacingUnit",
-            AliasPath("properties", "intendedTimeSpacingUnit"),
-        ),
-    )
-
-    @field_validator(
-        "phenomenon_begin_time",
-        "phenomenon_end_time",
-        "result_begin_time",
-        "result_end_time",
-        mode="before",
-    )
-    def split_time(cls, value: str, info) -> str:
-        if isinstance(value, str):
-            parts = value.split("/")
-            return parts[0] if "begin" in info.field_name else parts[-1]
-        return value
-
-
-class Datastream(HydroServerModel, DatastreamFields):
-    def __init__(
-        self,
-        _connection: "HydroServer",
-        _uid: Union[UUID, str],
-        **data,
-    ):
-        super().__init__(
-            _connection=_connection, _model_ref="datastreams", _uid=_uid, **data
-        )
+    ] = None
+    data_source_id: Optional[uuid.UUID] = None
+    thing_id: uuid.UUID
+    workspace_id: uuid.UUID
+    sensor_id: uuid.UUID
+    observed_property_id: uuid.UUID
+    processing_level_id: uuid.UUID
+    unit_id: uuid.UUID
+
+    _editable_fields: ClassVar[set[str]] = {
+        "name", "description", "observation_type", "sampled_medium", "no_data_value", "aggregation_statistic",
+        "time_aggregation_interval", "status", "result_type", "value_count", "phenomenon_begin_time",
+        "phenomenon_end_time", "result_begin_time", "result_end_time", "is_private", "is_visible",
+        "time_aggregation_interval_unit", "intended_time_spacing", "intended_time_spacing_unit", "thing_id",
+        "sensor_id", "observed_property_id", "processing_level_id", "unit_id"
+    }
+
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.datastreams, **data)
 
         self._workspace = None
         self._thing = None
@@ -147,14 +68,19 @@ class Datastream(HydroServerModel, DatastreamFields):
         self._unit = None
         self._processing_level = None
         self._sensor = None
+        self._data_source = None
+        self._data_archives = None
+
+    @classmethod
+    def get_route(cls):
+        return "datastreams"
 
     @property
     def workspace(self) -> "Workspace":
         """The workspace this datastream belongs to."""
 
         if self._workspace is None:
-            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
-            self._workspace = self._connection.workspaces.get(uid=datastream["workspaceId"])
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
 
         return self._workspace
 
@@ -163,179 +89,161 @@ class Datastream(HydroServerModel, DatastreamFields):
         """The thing this datastream belongs to."""
 
         if self._thing is None:
-            self._thing = self._connection.things.get(
-                uid=self.uid,
-                fetch_by_datastream_uid=True,
-            )
-            self._original_data["thing"] = self._thing
+            self._thing = self.client.things.get(uid=self.thing_id)
 
         return self._thing
 
     @thing.setter
-    def thing(self, thing: Union["Thing", UUID, str]):
+    def thing(self, thing: Union["Thing", UUID, str] = ...):
         if not thing:
             raise ValueError("Thing of datastream cannot be None.")
-        if str(getattr(thing, "uid", thing)) != str(self.thing.uid):
-            self._thing = self._connection.things.get(
-                uid=str(getattr(thing, "uid", thing))
-            )
+        if normalize_uuid(thing) != str(self.thing_id):
+            self.thing_id = normalize_uuid(thing)
+            self._thing = None
 
     @property
     def sensor(self) -> "Sensor":
-        """The sensor this datastream uses."""
+        """The sensor of this datastream."""
 
         if self._sensor is None:
-            self._sensor = self._connection.sensors.get(
-                uid=self.uid,
-                fetch_by_datastream_uid=True,
-            )
-            self._original_data["sensor"] = self._sensor
+            self._sensor = self.client.sensors.get(uid=self.sensor_id)
 
         return self._sensor
 
     @sensor.setter
-    def sensor(self, sensor: Union["Sensor", UUID, str]):
+    def sensor(self, sensor: Union["Sensor", UUID, str] = ...):
         if not sensor:
             raise ValueError("Sensor of datastream cannot be None.")
-        if str(getattr(sensor, "uid", sensor)) != str(self.sensor.uid):
-            self._sensor = self._connection.sensors.get(
-                uid=str(getattr(sensor, "uid", sensor))
-            )
+        if normalize_uuid(sensor) != str(self.sensor_id):
+            self.sensor_id = normalize_uuid(sensor)
+            self._sensor = None
 
     @property
-    def observed_property(self) -> "Thing":
+    def observed_property(self) -> "ObservedProperty":
         """The observed property of this datastream."""
 
         if self._observed_property is None:
-            self._observed_property = self._connection.observedproperties.get(
-                uid=self.uid,
-                fetch_by_datastream_uid=True,
-            )
-            self._original_data["observed_property"] = self._observed_property
+            self._observed_property = self.client.observedproperties.get(uid=self.observed_property_id)
 
         return self._observed_property
 
     @observed_property.setter
-    def observed_property(
-        self, observed_property: Union["ObservedProperty", UUID, str]
-    ):
+    def observed_property(self, observed_property: Union["ObservedProperty", UUID, str] = ...):
         if not observed_property:
             raise ValueError("Observed property of datastream cannot be None.")
-        if str(getattr(observed_property, "uid", observed_property)) != str(
-            self.observed_property.uid
-        ):
-            self._observed_property = self._connection.observedproperties.get(
-                uid=str(getattr(observed_property, "uid", observed_property))
-            )
+        if normalize_uuid(observed_property) != str(self.observed_property_id):
+            self.observed_property_id = normalize_uuid(observed_property)
+            self._observed_property = None
 
     @property
     def unit(self) -> "Unit":
-        """The unit this datastream uses."""
+        """The unit of this datastream."""
 
         if self._unit is None:
-            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
-            self._unit = self._connection.units.get(uid=datastream["unitId"])
-            self._original_data["unit"] = self._unit
+            self._unit = self.client.units.get(uid=self.unit_id)
 
         return self._unit
 
     @unit.setter
-    def unit(self, unit: Union["Unit", UUID, str]):
+    def unit(self, unit: Union["Unit", UUID, str] = ...):
         if not unit:
             raise ValueError("Unit of datastream cannot be None.")
-        if str(getattr(unit, "uid", unit)) != str(self.unit.uid):
-            self._unit = self._connection.units.get(uid=str(getattr(unit, "uid", unit)))
+        if normalize_uuid(unit) != str(self.unit_id):
+            self.unit_id = normalize_uuid(unit)
+            self._unit = None
 
     @property
-    def processing_level(self) -> "Thing":
+    def processing_level(self) -> "ProcessingLevel":
         """The processing level of this datastream."""
 
         if self._processing_level is None:
-            datastream = self._connection.request("get", f"/api/data/datastreams/{str(self.uid)}").json()
-            self._processing_level = self._connection.processinglevels.get(uid=datastream["processingLevelId"])
-            self._original_data["processing_level"] = self._processing_level
+            self._processing_level = self.client.processinglevels.get(uid=self.processing_level_id)
 
         return self._processing_level
 
     @processing_level.setter
-    def processing_level(self, processing_level: Union["ProcessingLevel", UUID, str]):
+    def processing_level(self, processing_level: Union["ProcessingLevel", UUID, str] = ...):
         if not processing_level:
             raise ValueError("Processing level of datastream cannot be None.")
-        if str(getattr(processing_level, "uid", processing_level)) != str(
-            self.processing_level.uid
-        ):
-            self._processing_level = self._connection.processinglevels.get(
-                uid=str(getattr(processing_level, "uid", processing_level))
-            )
+        if normalize_uuid(processing_level) != str(self.processing_level_id):
+            self.processing_level_id = normalize_uuid(processing_level)
+            self._processing_level = None
 
-    def refresh(self):
-        """Refresh this datastream from HydroServer."""
+    @property
+    def data_source(self) -> Optional["DataSource"]:
+        """The data source of this datastream."""
 
-        self._workspace = None
-        self._thing = None
-        self._observed_property = None
-        self._unit = None
-        self._processing_level = None
-        self._sensor = None
-        super()._refresh()
+        if self._data_source is None and self.data_source_id is not None:
+            self._data_source = self.client.datasources.get(uid=self.data_source_id)
 
-    def save(self):
-        """Save changes to this datastream to HydroServer."""
+        return self._data_source
 
-        super()._save()
+    @property
+    def data_archives(self) -> List["DataArchive"]:
+        """The data archives of this datastream."""
 
-    def delete(self):
-        """Delete this datastream from HydroServer."""
+        if self._data_archives is None:
+            self._data_archives = self.client.dataarchives.list(datastream=self.uid, fetch_all=True).items
 
-        super()._delete()
+        return self._data_archives
 
     def get_observations(
         self,
-        start_time: datetime = None,
-        end_time: datetime = None,
-        page: int = 1,
+        page: int = ...,
         page_size: int = 100000,
-        include_quality: bool = False,
+        order_by: List[str] = ...,
+        phenomenon_time_max: datetime = ...,
+        phenomenon_time_min: datetime = ...,
         fetch_all: bool = False,
-    ) -> DataFrame:
+    ) -> pd.DataFrame:
         """Retrieve the observations for this datastream."""
 
-        return self._connection.datastreams.get_observations(
+        return self.client.datastreams.get_observations(
             uid=self.uid,
-            start_time=start_time,
-            end_time=end_time,
             page=page,
             page_size=page_size,
-            include_quality=include_quality,
-            fetch_all=fetch_all,
+            order_by=order_by,
+            phenomenon_time_max=phenomenon_time_max,
+            phenomenon_time_min=phenomenon_time_min,
+            fetch_all=fetch_all
         )
 
     def load_observations(
         self,
-        observations: DataFrame,
+        observations: pd.DataFrame,
     ) -> None:
         """Load a DataFrame of observations to the datastream."""
 
-        return self._connection.datastreams.load_observations(
+        return self.client.datastreams.load_observations(
             uid=self.uid,
             observations=observations,
         )
 
+    def delete_observations(
+        self,
+        phenomenon_time_start: Optional[datetime] = None,
+        phenomenon_time_end: Optional[datetime] = None,
+    ):
+        """Delete the observations for this datastream."""
+
+        return self.client.datastreams.delete_observations(
+            uid=self.uid,
+            phenomenon_time_start=phenomenon_time_start,
+            phenomenon_time_end=phenomenon_time_end,
+        )
+
     # TODO: Find a better long-term solution for this issue.
     def sync_phenomenon_end_time(self):
         """Ensures the phenomenon_end_time field matches the actual end time of the observations."""
 
-        response = self._connection.request(
-            "get", f"/api/data/datastreams/{str(self.uid)}/observations",
-            params={
-                "order": "desc",
-                "page": 1,
-                "page_size": 1
-            }
+        path = f"/{self.client.base_route}/{self.get_route()}/{str(self.uid)}/observations"
+        response = self.client.request(
+            "get", path, params={"page_size": 1, "order_by": "-phenomenonTime"}
+
         ).json()
 
-        if len(response["phenomenon_time"]) > 0:
-            self.phenomenon_end_time = datetime.fromisoformat(response["phenomenon_time"][0])
+        if len(response) > 0:
+            self.phenomenon_end_time = datetime.fromisoformat(response[0]["phenomenonTime"])
         else:
             self.phenomenon_end_time = None
 

hydroserverpy/api/models/sta/observation.py

@@ -0,0 +1,101 @@
+import pandas as pd
+from typing import Optional, Any, List, TYPE_CHECKING
+from dataclasses import dataclass
+from requests import Response
+from pydantic.alias_generators import to_snake
+
+if TYPE_CHECKING:
+    from hydroserverpy.api.models import Datastream
+
+
+@dataclass
+class ObservationCollection:
+    dataframe: pd.DataFrame
+    filters: Optional[dict[str, Any]] = None
+    order_by: Optional[List[str]] = None
+    page: Optional[int] = None
+    page_size: Optional[int] = None
+    total_pages: Optional[int] = None
+    total_count: Optional[int] = None
+
+    def __init__(
+        self,
+        datastream: "Datastream",
+        response: Optional[Response] = None,
+        **data
+    ):
+        self.filters = data.get("filters")
+        self.order_by = data.get("order_by")
+        self.page = data.get("page") or (int(response.headers.get("X-Page")) if response else None)
+        self.page_size = data.get("page_size") or (int(response.headers.get("X-Page-Size")) if response else None)
+        self.total_pages = data.get("total_pages") or (int(response.headers.get("X-Total-Pages")) if response else None)
+        self.total_count = data.get("total_count") or (int(response.headers.get("X-Total-Count")) if response else None)
+        self.datastream = datastream
+
+        if "dataframe" in data:
+            self.dataframe = data["dataframe"]
+        elif response is not None:
+            data = response.json()
+            self.dataframe = pd.DataFrame({to_snake(k): v for k, v in data.items()})
+            if "phenomenon_time" in self.dataframe.columns:
+                self.dataframe["phenomenon_time"] = pd.to_datetime(self.dataframe["phenomenon_time"], utc=True)
+        else:
+            self.dataframe = pd.DataFrame()
+
+    def next_page(self):
+        """Fetches the next page of data from HydroServer."""
+
+        return self.datastream.get_observations(
+            **(self.filters or {}),
+            page=(self.page or 0) + 1,
+            page_size=self.page_size or 100000,
+            order_by=self.order_by or ...,
+        )
+
+    def previous_page(self):
+        """Fetches the previous page of data from HydroServer."""
+
+        if not self.page or self.page <= 1:
+            return None
+
+        return self.datastream.get_observations(
+            **(self.filters or {}),
+            page=self.page - 1,
+            page_size=self.page_size or 100000,
+            order_by=self.order_by or ...,
+        )
+
+    def fetch_all(self) -> "ObservationCollection":
+        """Fetches all pages of data from HydroServer for this collection."""
+
+        all_dataframes = []
+        page_num = 1
+
+        while self.total_pages is None or page_num <= self.total_pages:
+            if page_num == self.page:
+                all_dataframes.append(self.dataframe)
+            else:
+                observations = self.datastream.get_observations(
+                    **(self.filters or {}),
+                    page=page_num,
+                    page_size=self.page_size or 100000,
+                    order_by=self.order_by or ...,
+                )
+                if observations.dataframe.empty:
+                    break
+                all_dataframes.append(observations.dataframe)
+
+            page_num += 1
+
+        merged_dataframe = pd.concat(all_dataframes, ignore_index=True)
+
+        return self.__class__(
+            dataframe=merged_dataframe,
+            datastream=self.datastream,
+            filters=self.filters,
+            order_by=self.order_by or ...,
+            page=1,
+            page_size=len(merged_dataframe),
+            total_pages=1,
+            total_count=len(merged_dataframe)
+        )

hydroserverpy/api/models/sta/observed_property.py

@@ -1,72 +1,37 @@
-from typing import Union, TYPE_CHECKING
-from uuid import UUID
-from pydantic import BaseModel, Field, AliasChoices, AliasPath
-from ..base import HydroServerModel
+import uuid
+from typing import Optional, ClassVar, TYPE_CHECKING
+from pydantic import Field
+from ..base import HydroServerBaseModel
 
 if TYPE_CHECKING:
     from hydroserverpy import HydroServer
     from hydroserverpy.api.models import Workspace
 
 
-class ObservedPropertyFields(BaseModel):
+class ObservedProperty(HydroServerBaseModel):
     name: str = Field(..., max_length=255)
     definition: str
     description: str
-    observed_property_type: str = Field(
-        ...,
-        max_length=255,
-        serialization_alias="type",
-        validation_alias=AliasChoices("type", AliasPath("properties", "variableType")),
-    )
-    code: str = Field(
-        ...,
-        max_length=255,
-        validation_alias=AliasChoices("code", AliasPath("properties", "variableCode")),
-    )
+    observed_property_type: str = Field(..., max_length=255, alias="type")
+    code: str = Field(..., max_length=255)
+    workspace_id: Optional[uuid.UUID] = None
 
+    _editable_fields: ClassVar[set[str]] = {"name", "definition", "description", "observed_property_type", "code"}
 
-class ObservedProperty(HydroServerModel, ObservedPropertyFields):
-    def __init__(self, _connection: "HydroServer", _uid: Union[UUID, str], **data):
-        super().__init__(
-            _connection=_connection, _model_ref="observedproperties", _uid=_uid, **data
-        )
-
-        self._workspace_id = (
-            data.get("workspace_id")
-            or data.get("workspaceId")
-            or (
-                None
-                if data.get("properties", {}).get("workspace") is None
-                else data.get("properties", {}).get("workspace", {}).get("id")
-            )
-        )
-        self._workspace_id = (
-            str(self._workspace_id) if self._workspace_id is not None else None
-        )
+    def __init__(self, client: "HydroServer", **data):
+        super().__init__(client=client, service=client.observedproperties, **data)
 
         self._workspace = None
 
+    @classmethod
+    def get_route(cls):
+        return "observed-properties"
+
     @property
-    def workspace(self) -> "Workspace":
+    def workspace(self) -> Optional["Workspace"]:
         """The workspace this observed property belongs to."""
 
-        if self._workspace is None and self._workspace_id:
-            self._workspace = self._connection.workspaces.get(uid=self._workspace_id)
+        if self._workspace is None and self.workspace_id:
+            self._workspace = self.client.workspaces.get(uid=self.workspace_id)
 
         return self._workspace
-
-    def refresh(self):
-        """Refresh this observed property from HydroServer."""
-
-        super()._refresh()
-        self._workspace = None
-
-    def save(self):
-        """Save changes to this observed property to HydroServer."""
-
-        super()._save()
-
-    def delete(self):
-        """Delete this observed property from HydroServer."""
-
-        super()._delete()