hydroserverpy 0.3.0__py3-none-any.whl → 0.5.0b1__py3-none-any.whl

hydroserverpy/api/services/sta/sensor.py

@@ -0,0 +1,116 @@
+from typing import Optional, Union, List, TYPE_CHECKING
+from uuid import UUID
+from ..base import SensorThingsService
+from hydroserverpy.api.models import Sensor
+
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class SensorService(SensorThingsService):
+    def __init__(self, connection: "HydroServer"):
+        self._model = Sensor
+        self._api_route = "api/data"
+        self._endpoint_route = "sensors"
+        self._sta_route = "api/sensorthings/v1.1/Sensors"
+
+        super().__init__(connection)
+
+    def list(
+        self,
+        workspace: Optional[Union["Workspace", UUID, str]] = None,
+        page: int = 1,
+        page_size: int = 100,
+    ) -> List["Sensor"]:
+        """Fetch a collection of sensors."""
+
+        params = {"$top": page_size, "$skip": page_size * (page - 1)}
+
+        if workspace:
+            params["$filter"] = (
+                f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
+            )
+
+        return super()._list(params=params)
+
+    def get(
+        self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
+    ) -> "Sensor":
+        """Get a sensor by ID."""
+
+        return self._get(
+            uid=str(uid),
+            path=(
+                f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Sensor"
+                if fetch_by_datastream_uid
+                else None
+            ),
+        )
+
+    def create(
+        self,
+        workspace: Union["Workspace", UUID, str],
+        name: str,
+        description: str,
+        encoding_type: str,
+        method_type: str,
+        manufacturer: Optional[str] = None,
+        sensor_model: Optional[str] = None,
+        sensor_model_link: Optional[str] = None,
+        method_link: Optional[str] = None,
+        method_code: Optional[str] = None,
+    ) -> "Sensor":
+        """Create a new sensor."""
+
+        kwargs = {
+            "name": name,
+            "description": description,
+            "encodingType": encoding_type,
+            "methodType": method_type,
+            "manufacturer": manufacturer,
+            "model": sensor_model,
+            "modelLink": sensor_model_link,
+            "methodLink": method_link,
+            "methodCode": method_code,
+            "workspaceId": str(getattr(workspace, "uid", workspace)),
+        }
+
+        return super()._create(**kwargs)
+
+    def update(
+        self,
+        uid: Union[UUID, str],
+        name: str = ...,
+        description: str = ...,
+        encoding_type: str = ...,
+        method_type: str = ...,
+        manufacturer: Optional[str] = ...,
+        sensor_model: Optional[str] = ...,
+        sensor_model_link: Optional[str] = ...,
+        method_link: Optional[str] = ...,
+        method_code: Optional[str] = ...,
+    ) -> "Sensor":
+        """Update a sensor."""
+
+        kwargs = {
+            "name": name,
+            "description": description,
+            "encodingType": encoding_type,
+            "methodType": method_type,
+            "manufacturer": manufacturer,
+            "model": sensor_model,
+            "modelLink": sensor_model_link,
+            "methodLink": method_link,
+            "methodCode": method_code,
+        }
+
+        return super()._update(
+            uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
+        )
+
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """Delete a sensor."""
+
+        super()._delete(uid=str(uid))

hydroserverpy/api/services/sta/thing.py

@@ -0,0 +1,188 @@
+import json
+from typing import TYPE_CHECKING, Union, IO, List, Dict, Optional
+from uuid import UUID
+from ..base import SensorThingsService
+from hydroserverpy.api.models import Thing
+
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class ThingService(SensorThingsService):
+    def __init__(self, connection: "HydroServer"):
+        self._model = Thing
+        self._api_route = "api/data"
+        self._endpoint_route = "things"
+        self._sta_route = "api/sensorthings/v1.1/Things"
+
+        super().__init__(connection)
+
+    def list(
+        self,
+        workspace: Optional[Union["Workspace", UUID, str]] = None,
+        page: int = 1,
+        page_size: int = 100,
+    ) -> List["Thing"]:
+        """Fetch a collection of things."""
+
+        params = {
+            "$top": page_size,
+            "$skip": page_size * (page - 1),
+            "$expand": "Locations",
+        }
+
+        if workspace:
+            params["$filter"] = (
+                f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
+            )
+
+        return super()._list(params=params)
+
+    def get(
+        self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
+    ) -> "Thing":
+        """Get a thing by ID."""
+
+        params = {"$expand": "Locations"}
+        return self._get(
+            uid=str(uid),
+            path=(
+                f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Thing"
+                if fetch_by_datastream_uid
+                else None
+            ),
+            params=params,
+        )
+
+    def create(
+        self,
+        workspace: Union["Workspace", UUID, str],
+        name: str,
+        description: str,
+        sampling_feature_type: str,
+        sampling_feature_code: str,
+        site_type: str,
+        is_private: False,
+        latitude: float,
+        longitude: float,
+        elevation_m: Optional[float] = None,
+        elevation_datum: Optional[str] = None,
+        state: Optional[str] = None,
+        county: Optional[str] = None,
+        country: Optional[str] = None,
+        data_disclaimer: Optional[str] = None,
+    ) -> "Thing":
+        """Create a new thing."""
+
+        kwargs = {
+            "name": name,
+            "description": description,
+            "samplingFeatureType": sampling_feature_type,
+            "samplingFeatureCode": sampling_feature_code,
+            "siteType": site_type,
+            "isPrivate": is_private,
+            "latitude": latitude,
+            "longitude": longitude,
+            "elevation_m": elevation_m,
+            "elevationDatum": elevation_datum,
+            "state": state,
+            "county": county,
+            "country": country,
+            "dataDisclaimer": data_disclaimer,
+            "workspaceId": str(getattr(workspace, "uid", workspace)),
+        }
+
+        return super()._create(**kwargs)
+
+    def update(
+        self,
+        uid: Union[UUID, str],
+        name: str = ...,
+        description: str = ...,
+        sampling_feature_type: str = ...,
+        sampling_feature_code: str = ...,
+        site_type: str = ...,
+        is_private: False = ...,
+        latitude: float = ...,
+        longitude: float = ...,
+        elevation_m: Optional[float] = ...,
+        elevation_datum: Optional[str] = ...,
+        state: Optional[str] = ...,
+        county: Optional[str] = ...,
+        country: Optional[str] = ...,
+        data_disclaimer: Optional[str] = ...,
+    ) -> "Thing":
+        """Update a thing."""
+
+        kwargs = {
+            "name": name,
+            "description": description,
+            "samplingFeatureType": sampling_feature_type,
+            "samplingFeatureCode": sampling_feature_code,
+            "siteType": site_type,
+            "isPrivate": is_private,
+            "latitude": latitude,
+            "longitude": longitude,
+            "elevation_m": elevation_m,
+            "elevationDatum": elevation_datum,
+            "state": state,
+            "county": county,
+            "country": country,
+            "dataDisclaimer": data_disclaimer,
+        }
+
+        return super()._update(
+            uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
+        )
+
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """Delete a thing."""
+
+        super()._delete(uid=str(uid))
+
+    def add_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
+        """Tag a HydroServer thing."""
+
+        return self._connection.request(
+            "post",
+            f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
+            data=json.dumps({"key": key, "value": value}),
+        ).json()
+
+    def update_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
+        """Update the tag of a HydroServer thing."""
+
+        return self._connection.request(
+            "put",
+            f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
+            data=json.dumps({"key": key, "value": value}),
+        ).json()
+
+    def delete_tag(self, uid: Union[UUID, str], key: str) -> None:
+        """Remove a tag from a HydroServer thing."""
+
+        self._connection.request(
+            "delete",
+            f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
+            data=json.dumps({"key": key}),
+        )
+
+    def add_photo(self, uid: Union[UUID, str], file: IO[bytes]) -> Dict[str, str]:
+        """Add a photo of a HydroServer thing."""
+
+        return self._connection.request(
+            "post",
+            f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
+            files={"file": file},
+        ).json()
+
+    def delete_photo(self, uid: Union[UUID, str], name: str) -> None:
+        """Delete a photo of a HydroServer thing."""
+
+        self._connection.request(
+            "delete",
+            f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
+            data=json.dumps({"name": name}),
+        )

hydroserverpy/api/services/sta/unit.py

@@ -0,0 +1,82 @@
+from typing import Optional, Union, List, TYPE_CHECKING
+from uuid import UUID
+from ..base import EndpointService
+from hydroserverpy.api.models import Unit
+
+
+if TYPE_CHECKING:
+    from hydroserverpy import HydroServer
+    from hydroserverpy.api.models import Workspace
+
+
+class UnitService(EndpointService):
+    def __init__(self, connection: "HydroServer"):
+        self._model = Unit
+        self._api_route = "api/data"
+        self._endpoint_route = "units"
+
+        super().__init__(connection)
+
+    def list(
+        self,
+        workspace: Optional[Union["Workspace", UUID, str]] = None,
+    ) -> List["Unit"]:
+        """Fetch a collection of units."""
+
+        workspace_id = getattr(workspace, "uid", workspace)
+        workspace_id = str(workspace_id) if workspace_id else None
+
+        return super()._list(
+            params={"workspace_id": workspace_id} if workspace_id else {},
+        )
+
+    def get(self, uid: Union[UUID, str]) -> "Unit":
+        """Get a unit by ID."""
+
+        return super()._get(uid=str(uid))
+
+    def create(
+        self,
+        workspace: Union["Workspace", UUID, str],
+        name: str,
+        symbol: str,
+        definition: str,
+        unit_type: str,
+    ) -> "Unit":
+        """Create a new unit."""
+
+        kwargs = {
+            "name": name,
+            "symbol": symbol,
+            "definition": definition,
+            "type": unit_type,
+            "workspaceId": str(getattr(workspace, "uid", workspace)),
+        }
+
+        return super()._create(**kwargs)
+
+    def update(
+        self,
+        uid: Union[UUID, str],
+        name: str = ...,
+        symbol: str = ...,
+        definition: str = ...,
+        unit_type: str = ...,
+    ) -> "Unit":
+        """Update a unit."""
+
+        kwargs = {
+            "name": name,
+            "symbol": symbol,
+            "definition": definition,
+            "type": unit_type,
+        }
+
+        return super()._update(
+            uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
+        )
+
+    def delete(self, uid: Union[UUID, str]) -> None:
+        """Delete a unit."""
+
+        super()._delete(uid=str(uid))

hydroserverpy/etl/__init__.py

@@ -0,0 +1,21 @@
+from .extractors.local_file_extractor import LocalFileExtractor
+from .extractors.ftp_extractor import FTPExtractor
+from .extractors.http_extractor import HTTPExtractor
+from .transformers.csv_transformer import CSVTransformer
+from .transformers.json_transformer import JSONTransformer
+from .transformers.base import Transformer
+from .extractors.base import Extractor
+from .loaders.base import Loader
+from .loaders.hydroserver_loader import HydroServerLoader
+
+__all__ = [
+    "CSVTransformer",
+    "JSONTransformer",
+    "LocalFileExtractor",
+    "FTPExtractor",
+    "HTTPExtractor",
+    "Extractor",
+    "Transformer",
+    "Loader",
+    "HydroServerLoader",
+]

hydroserverpy/etl/extractors/base.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+from ..types import TimeRange
+
+
+class Extractor(ABC):
+    @abstractmethod
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    @abstractmethod
+    def extract(self):
+        pass

hydroserverpy/etl/extractors/ftp_extractor.py

@@ -0,0 +1,50 @@
+import logging
+from ftplib import FTP
+from io import BytesIO
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class FTPExtractor(Extractor):
+    def __init__(
+        self,
+        host: str,
+        filepath: str,
+        username: str = None,
+        password: str = None,
+        port: int = 21,
+    ):
+        self.host = host
+        self.port = int(port)
+        self.username = username
+        self.password = password
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Downloads the file from the FTP server and returns a file-like object.
+        """
+        ftp = FTP()
+        try:
+            ftp.connect(self.host, self.port)
+            ftp.login(user=self.username, passwd=self.password)
+            logging.info(f"Connected to FTP server: {self.host}:{self.port}")
+
+            data = BytesIO()
+            ftp.retrbinary(f"RETR {self.filepath}", data.write)
+            logging.info(
+                f"Successfully downloaded file '{self.filepath}' from FTP server."
+            )
+            data.seek(0)
+            return data
+        except Exception as e:
+            logging.error(f"Error retrieving file from FTP server: {e}")
+            return None
+        finally:
+            if ftp:
+                ftp.quit()

hydroserverpy/etl/extractors/http_extractor.py

@@ -0,0 +1,84 @@
+import logging
+from hydroserverpy.etl.types import TimeRange
+import requests
+from io import BytesIO
+from typing import Dict
+from .base import Extractor
+
+
+class HTTPExtractor(Extractor):
+    def __init__(
+        self,
+        url: str,
+        url_variables: dict = None,
+        params: dict = None,
+        headers: dict = None,
+        auth: tuple = None,
+    ):
+        self.url = self.format_url(url, url_variables or {})
+        self.params = params
+        self.headers = headers
+        self.auth = auth
+        self.start_date = None
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        start_times = [
+            req["start_time"] for req in data_requirements.values() if req["start_time"]
+        ]
+
+        if start_times:
+            oldest_start_time = min(start_times).isoformat()
+            start_time_key = self.params.pop("start_time_key", None)
+            if start_time_key:
+                self.params[start_time_key] = oldest_start_time
+                logging.info(
+                    f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
+                )
+            else:
+                logging.warning("'start_time_key' not found in params.")
+
+        end_times = [
+            req["end_time"] for req in data_requirements.values() if req["end_time"]
+        ]
+
+        if end_times:
+            newest_end_time = max(end_times).isoformat()
+            end_time_key = self.params.pop("end_time_key", None)
+            if end_time_key:
+                self.params[end_time_key] = newest_end_time
+                logging.info(
+                    f"Set end_time to {newest_end_time} and removed 'end_time_key'"
+                )
+            else:
+                logging.warning("'end_time_key' not found in params.")
+
+    def extract(self):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+        response = requests.get(
+            url=self.url,
+            params=self.params,
+            headers=self.headers,
+            auth=self.auth,
+            stream=True,
+        )
+        response.raise_for_status()
+        logging.info(f"Successfully downloaded file from {response.url}")
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data
+
+    @staticmethod
+    def format_url(url_template, url_variables):
+        try:
+            url = url_template.format(**url_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing configuration url_variable: {missing_key}")
+
+        return url

hydroserverpy/etl/extractors/local_file_extractor.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, filepath: str):
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.filepath, "r")
+            logging.info(f"Successfully opened file '{self.filepath}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.filepath}': {e}")
+            return None

hydroserverpy/etl/hydroserver_etl.py

@@ -0,0 +1,40 @@
+import logging
+import pandas as pd
+
+
+class HydroServerETL:
+    def __init__(self, extractor, transformer, loader, source_target_map):
+        self.extractor = extractor
+        self.transformer = transformer
+        self.loader = loader
+        self.source_target_map = source_target_map
+
+    def run(self):
+        """
+        Extracts, transforms, and loads data as defined by the class parameters.
+        """
+
+        # Step 1: Get Target System data requirements from the Loader & prepare parameters for the Extractor
+        data_requirements = self.loader.get_data_requirements(self.source_target_map)
+        self.extractor.prepare_params(data_requirements)
+
+        # Step 2: Extract
+        data = self.extractor.extract()
+        if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+            logging.warning(f"No data was returned from the extractor. Ending ETL run.")
+            return
+        else:
+            logging.info(f"Successfully extracted data.")
+
+        # Step 3: Transform
+        if self.transformer:
+            data = self.transformer.transform(data)
+            if data is None or (isinstance(data, pd.DataFrame) and data.empty):
+                logging.warning(f"No data returned from the transformer. Ending run.")
+                return
+            else:
+                logging.info(f"Successfully transformed data. {data}")
+
+        # Step 4: Load
+        self.loader.load(data, self.source_target_map)
+        logging.info("Successfully loaded data.")

hydroserverpy/etl/loaders/base.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+import pandas as pd
+
+
+class Loader(ABC):
+    @abstractmethod
+    def load(self, *args, **kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def get_data_requirements(self, df: pd.DataFrame) -> Dict[str, pd.Timestamp]:
+        pass