hydroserverpy 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

hydroserverpy/core/schemas/sensors.py

@@ -5,40 +5,55 @@ from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
 class SensorFields(BaseModel):
     name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the sensor.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The name of the sensor.",
     )
     description: str = Field(
-        strip_whitespace=True,
-        description='A description of the sensor.'
+        strip_whitespace=True, description="A description of the sensor."
     )
     encoding_type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The encoding type of the sensor.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The encoding type of the sensor.",
     )
     manufacturer: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The manufacturer of the sensor.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The manufacturer of the sensor.",
     )
     model: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='The model of the sensor.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="The model of the sensor.",
     )
     model_link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=500,
-        description='A link to a website or file that describes the sensor model.'
+        None,
+        strip_whitespace=True,
+        max_length=500,
+        description="A link to a website or file that describes the sensor model.",
     )
     method_type: str = Field(
-        ..., strip_whitespace=True, max_length=100,
-        description='The type of method used by this sensor to collect observations.'
+        ...,
+        strip_whitespace=True,
+        max_length=100,
+        description="The type of method used by this sensor to collect observations.",
     )
     method_link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=500,
-        description='A link to a website or file that describes the sensor method.'
+        None,
+        strip_whitespace=True,
+        max_length=500,
+        description="A link to a website or file that describes the sensor method.",
     )
     method_code: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=50,
-        description='A code representing the sensor method.'
+        None,
+        strip_whitespace=True,
+        max_length=50,
+        description="A code representing the sensor method.",
     )
 
     model_config = ConfigDict(protected_namespaces=())

hydroserverpy/core/schemas/things.py

@@ -10,73 +10,106 @@ if TYPE_CHECKING:
 
 class ThingFields(BaseModel):
     name: str = Field(
-        ..., strip_whitespace=True, max_length=200,
-        description='The name of the site/thing.'
+        ...,
+        strip_whitespace=True,
+        max_length=200,
+        description="The name of the site/thing.",
     )
     description: str = Field(
-        ..., strip_whitespace=True,
-        description='A description of the site/thing.'
+        ..., strip_whitespace=True, description="A description of the site/thing."
     )
     sampling_feature_type: str = Field(
-        ..., strip_whitespace=True, max_length=200,
-        description='The sampling feature type of the site/thing.'
+        ...,
+        strip_whitespace=True,
+        max_length=200,
+        description="The sampling feature type of the site/thing.",
     )
     sampling_feature_code: str = Field(
-        ..., strip_whitespace=True, max_length=200,
-        description='A code representing the sampling feature of the site/thing.'
+        ...,
+        strip_whitespace=True,
+        max_length=200,
+        description="A code representing the sampling feature of the site/thing.",
     )
     site_type: str = Field(
-        ..., strip_whitespace=True, max_length=200,
-        description='The type of the site/thing.'
+        ...,
+        strip_whitespace=True,
+        max_length=200,
+        description="The type of the site/thing.",
     )
     data_disclaimer: Optional[str] = Field(
-        None, strip_whitespace=True,
-        description='An optional data disclaimer to attach to observations collected at this site/thing.'
+        None,
+        strip_whitespace=True,
+        description="An optional data disclaimer to attach to observations collected at this site/thing.",
     )
 
 
 # Get a list of all ISO 3166-1 alpha-2 country codes
-valid_country_codes = [code for code, _ in countries_for_language('en')]
+valid_country_codes = [code for code, _ in countries_for_language("en")]
 
 
 class LocationFields(BaseModel):
     latitude: float = Field(
-        ..., ge=-90, le=90, serialization_alias='latitude',
-        validation_alias=AliasChoices('latitude', AliasPath('location', 'latitude')),
-        description='The WGS84 latitude of the location.'
+        ...,
+        ge=-90,
+        le=90,
+        serialization_alias="latitude",
+        validation_alias=AliasChoices("latitude", AliasPath("location", "latitude")),
+        description="The WGS84 latitude of the location.",
     )
     longitude: float = Field(
-        ..., ge=-180, le=180, serialization_alias='longitude',
-        validation_alias=AliasChoices('longitude', AliasPath('location', 'longitude')),
-        description='The WGS84 longitude of the location.'
+        ...,
+        ge=-180,
+        le=180,
+        serialization_alias="longitude",
+        validation_alias=AliasChoices("longitude", AliasPath("location", "longitude")),
+        description="The WGS84 longitude of the location.",
     )
     elevation_m: Optional[float] = Field(
-        None, ge=-99999, le=99999, serialization_alias='elevation_m',
-        validation_alias=AliasChoices('elevation_m', AliasPath('location', 'elevation_m')),
-        description='The elevation in meters of the location.'
+        None,
+        ge=-99999,
+        le=99999,
+        serialization_alias="elevation_m",
+        validation_alias=AliasChoices(
+            "elevation_m", AliasPath("location", "elevation_m")
+        ),
+        description="The elevation in meters of the location.",
     )
     elevation_datum: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255, serialization_alias='elevationDatum',
-        validation_alias=AliasChoices('elevationDatum', AliasPath('location', 'elevationDatum')),
-        description='The datum used to represent the elevation of the location.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        serialization_alias="elevationDatum",
+        validation_alias=AliasChoices(
+            "elevationDatum", AliasPath("location", "elevationDatum")
+        ),
+        description="The datum used to represent the elevation of the location.",
     )
     state: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=200, serialization_alias='state',
-        validation_alias=AliasChoices('state', AliasPath('location', 'state')),
-        description='The state/province of the location.'
+        None,
+        strip_whitespace=True,
+        max_length=200,
+        serialization_alias="state",
+        validation_alias=AliasChoices("state", AliasPath("location", "state")),
+        description="The state/province of the location.",
     )
     county: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=200, serialization_alias='county',
-        validation_alias=AliasChoices('county', AliasPath('location', 'county')),
-        description='The county/district of the location.'
+        None,
+        strip_whitespace=True,
+        max_length=200,
+        serialization_alias="county",
+        validation_alias=AliasChoices("county", AliasPath("location", "county")),
+        description="The county/district of the location.",
     )
     country: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=2, serialization_alias='country',
-        validation_alias=AliasChoices('country', AliasPath('location', 'country')),
-        description='The ISO 3166-1 alpha-2 country code of the location.'
+        None,
+        strip_whitespace=True,
+        max_length=2,
+        serialization_alias="country",
+        validation_alias=AliasChoices("country", AliasPath("location", "country")),
+        description="The ISO 3166-1 alpha-2 country code of the location.",
     )
 
-    @field_validator('country', mode='after')
+    @field_validator("country", mode="after")
     def check_country_code(cls, value: str) -> str:
         """
         Validate the country code to ensure it is an ISO 3166-1 alpha-2 country code.
@@ -89,7 +122,9 @@ class LocationFields(BaseModel):
         """
 
         if value and value.upper() not in valid_country_codes:
-            raise ValueError(f'Invalid country code: {value}. Must be an ISO 3166-1 alpha-2 country code.')
+            raise ValueError(
+                f"Invalid country code: {value}. Must be an ISO 3166-1 alpha-2 country code."
+            )
 
         return value
 
@@ -123,7 +158,7 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         self._archive = None
 
     @property
-    def datastreams(self) -> List['Datastream']:
+    def datastreams(self) -> List["Datastream"]:
         """
         The datastreams associated with the thing. If not already cached, fetch the datastreams from the
         server.
@@ -138,7 +173,7 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         return self._datastreams
 
     @property
-    def tags(self) -> List['Tag']:
+    def tags(self) -> List["Tag"]:
         """
         The tags associated with the thing. If not already cached, fetch the tags from the server.
 
@@ -152,7 +187,7 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         return self._tags
 
     @property
-    def photos(self) -> List['Photo']:
+    def photos(self) -> List["Photo"]:
         """
         The photos associated with the thing. If not already cached, fetch the photos from the server.
 
@@ -166,7 +201,7 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         return self._photos
 
     @property
-    def archive(self) -> 'Archive':
+    def archive(self) -> "Archive":
         """
         The archive associated with the thing. If not already cached, fetch the archive from the server.
 
@@ -185,7 +220,7 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         if they were previously loaded.
         """
 
-        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=['uid'])
+        entity = self._endpoint.get(uid=self.uid).model_dump(exclude=["uid"])
         self._original_data = entity
         self.__dict__.update(entity)
         if self._datastreams is not None:
@@ -218,7 +253,9 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
         """
 
         selected_tag = next((tag for tag in self._tags if tag.key == key))
-        updated_tag = self._endpoint.update_tag(uid=self.uid, tag_uid=selected_tag.uid, value=value)
+        updated_tag = self._endpoint.update_tag(
+            uid=self.uid, tag_uid=selected_tag.uid, value=value
+        )
         self._tags = [tag if tag.key != key else updated_tag for tag in self._tags]
 
     def delete_tag(self, key: str) -> None:
@@ -253,7 +290,9 @@ class Thing(HydroServerCoreModel, ThingFields, LocationFields):
 
         selected_photo = next((photo for photo in self._photos if photo.link == link))
         self._endpoint.delete_photo(uid=self.uid, photo_uid=selected_photo.uid)
-        self._photos = [photo for photo in self._photos if photo.link != selected_photo.link]
+        self._photos = [
+            photo for photo in self._photos if photo.link != selected_photo.link
+        ]
 
 
 class Archive(HydroServerBaseModel):
@@ -262,20 +301,24 @@ class Archive(HydroServerBaseModel):
     """
 
     link: Optional[str] = Field(
-        None, strip_whitespace=True, max_length=255,
-        description='A link to the HydroShare resource containing the archived site/thing.'
+        None,
+        strip_whitespace=True,
+        max_length=255,
+        description="A link to the HydroShare resource containing the archived site/thing.",
     )
-    frequency: Optional[Literal['daily', 'weekly', 'monthly']] = Field(
+    frequency: Optional[Literal["daily", "weekly", "monthly"]] = Field(
         ...,
-        description='The frequency at which the site/thing should be archived.',
+        description="The frequency at which the site/thing should be archived.",
     )
     path: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The path within the HydroShare resource containing the archived data.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The path within the HydroShare resource containing the archived data.",
     )
     datastream_ids: List[UUID] = Field(
         ...,
-        description='The list of datastreams that are included in the archived data.',
+        description="The list of datastreams that are included in the archived data.",
     )
 
 
@@ -285,12 +328,10 @@ class Tag(HydroServerBaseModel):
     """
 
     key: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The key of the tag.'
+        ..., strip_whitespace=True, max_length=255, description="The key of the tag."
     )
     value: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The value of the tag.'
+        ..., strip_whitespace=True, max_length=255, description="The value of the tag."
     )
 
 
@@ -300,10 +341,6 @@ class Photo(HydroServerBaseModel):
     """
 
     file_path: str = Field(
-        ..., strip_whitespace=True,
-        description='The file path of the photo.'
-    )
-    link: str = Field(
-        ..., strip_whitespace=True,
-        description='The link to the photo.'
+        ..., strip_whitespace=True, description="The file path of the photo."
     )
+    link: str = Field(..., strip_whitespace=True, description="The link to the photo.")

hydroserverpy/core/schemas/units.py

@@ -4,20 +4,19 @@ from hydroserverpy.core.schemas.base import HydroServerCoreModel
 
 class UnitFields:
     name: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The name of the unit.'
+        ..., strip_whitespace=True, max_length=255, description="The name of the unit."
     )
     symbol: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The symbol of the unit.'
+        ...,
+        strip_whitespace=True,
+        max_length=255,
+        description="The symbol of the unit.",
     )
     definition: str = Field(
-        ..., strip_whitespace=True,
-        description='The definition of the unit.'
+        ..., strip_whitespace=True, description="The definition of the unit."
     )
     type: str = Field(
-        ..., strip_whitespace=True, max_length=255,
-        description='The type of the unit.'
+        ..., strip_whitespace=True, max_length=255, description="The type of the unit."
     )
 
 

hydroserverpy/core/service.py

@@ -1,8 +1,16 @@
 import requests
 from typing import Optional
-from hydroserverpy.core.endpoints import (DataLoaderEndpoint, DataSourceEndpoint, DatastreamEndpoint, ThingEndpoint,
-                                          SensorEndpoint, UnitEndpoint, ProcessingLevelEndpoint,
-                                          ObservedPropertyEndpoint, ResultQualifierEndpoint)
+from hydroserverpy.core.endpoints import (
+    DataLoaderEndpoint,
+    DataSourceEndpoint,
+    DatastreamEndpoint,
+    ThingEndpoint,
+    SensorEndpoint,
+    UnitEndpoint,
+    ProcessingLevelEndpoint,
+    ObservedPropertyEndpoint,
+    ResultQualifierEndpoint,
+)
 
 
 class HydroServer:
@@ -22,16 +30,23 @@ class HydroServer:
     """
 
     def __init__(
-            self,
-            host: str,
-            username: Optional[str] = None,
-            password: Optional[str] = None,
-            apikey: Optional[str] = None,
-            api_route: str = 'api'
+        self,
+        host: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+        api_route: str = "api",
     ):
-        self.host = host.strip('/')
-        self.auth = (username or '__key__', password or apikey,) if (username and password) or apikey else None
-        self.api_route = api_route.strip('/')
+        self.host = host.strip("/")
+        self.auth = (
+            (
+                username or "__key__",
+                password or apikey,
+            )
+            if (username and password) or apikey
+            else None
+        )
+        self.api_route = api_route.strip("/")
         self._session = None
         self._timeout = 60
         self._initialize_session()
@@ -49,10 +64,8 @@ class HydroServer:
 
         self._session = requests.Session()
 
-        if self.auth and self.auth[0] == '__key__':
-            self._session.headers.update(
-                {'key': self.auth[1]}
-            )
+        if self.auth and self.auth[0] == "__key__":
+            self._session.headers.update({"key": self.auth[1]})
         elif self.auth:
             self._session.auth = self.auth
 
@@ -75,7 +88,8 @@ class HydroServer:
                 response = getattr(self._session, method)(
                     f'{self.host}/{path.strip("/")}',
                     timeout=self._timeout,
-                    *args, **kwargs
+                    *args,
+                    **kwargs,
                 )
                 response.raise_for_status()
                 return response

hydroserverpy/etl/__init__.py

@@ -0,0 +1,21 @@
+from .extractors.local_file_extractor import LocalFileExtractor
+from .extractors.ftp_extractor import FTPExtractor
+from .extractors.http_extractor import HTTPExtractor
+from .transformers.csv_transformer import CSVTransformer
+from .transformers.json_transformer import JSONTransformer
+from .transformers.base import Transformer
+from .extractors.base import Extractor
+from .loaders.base import Loader
+from .loaders.hydroserver_loader import HydroServerLoader
+
+__all__ = [
+    "CSVTransformer",
+    "JSONTransformer",
+    "LocalFileExtractor",
+    "FTPExtractor",
+    "HTTPExtractor",
+    "Extractor",
+    "Transformer",
+    "Loader",
+    "HydroServerLoader",
+]

hydroserverpy/etl/extractors/base.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+from ..types import TimeRange
+
+
+class Extractor(ABC):
+    @abstractmethod
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    @abstractmethod
+    def extract(self):
+        pass

hydroserverpy/etl/extractors/ftp_extractor.py

@@ -0,0 +1,50 @@
+import logging
+from ftplib import FTP
+from io import BytesIO
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class FTPExtractor(Extractor):
+    def __init__(
+        self,
+        host: str,
+        filepath: str,
+        username: str = None,
+        password: str = None,
+        port: int = 21,
+    ):
+        self.host = host
+        self.port = int(port)
+        self.username = username
+        self.password = password
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Downloads the file from the FTP server and returns a file-like object.
+        """
+        ftp = FTP()
+        try:
+            ftp.connect(self.host, self.port)
+            ftp.login(user=self.username, passwd=self.password)
+            logging.info(f"Connected to FTP server: {self.host}:{self.port}")
+
+            data = BytesIO()
+            ftp.retrbinary(f"RETR {self.filepath}", data.write)
+            logging.info(
+                f"Successfully downloaded file '{self.filepath}' from FTP server."
+            )
+            data.seek(0)
+            return data
+        except Exception as e:
+            logging.error(f"Error retrieving file from FTP server: {e}")
+            return None
+        finally:
+            if ftp:
+                ftp.quit()

hydroserverpy/etl/extractors/http_extractor.py

@@ -0,0 +1,84 @@
+import logging
+from hydroserverpy.etl.types import TimeRange
+import requests
+from io import BytesIO
+from typing import Dict
+from .base import Extractor
+
+
+class HTTPExtractor(Extractor):
+    def __init__(
+        self,
+        url: str,
+        url_variables: dict = None,
+        params: dict = None,
+        headers: dict = None,
+        auth: tuple = None,
+    ):
+        self.url = self.format_url(url, url_variables or {})
+        self.params = params
+        self.headers = headers
+        self.auth = auth
+        self.start_date = None
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        start_times = [
+            req["start_time"] for req in data_requirements.values() if req["start_time"]
+        ]
+
+        if start_times:
+            oldest_start_time = min(start_times).isoformat()
+            start_time_key = self.params.pop("start_time_key", None)
+            if start_time_key:
+                self.params[start_time_key] = oldest_start_time
+                logging.info(
+                    f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
+                )
+            else:
+                logging.warning("'start_time_key' not found in params.")
+
+        end_times = [
+            req["end_time"] for req in data_requirements.values() if req["end_time"]
+        ]
+
+        if end_times:
+            newest_end_time = max(end_times).isoformat()
+            end_time_key = self.params.pop("end_time_key", None)
+            if end_time_key:
+                self.params[end_time_key] = newest_end_time
+                logging.info(
+                    f"Set end_time to {newest_end_time} and removed 'end_time_key'"
+                )
+            else:
+                logging.warning("'end_time_key' not found in params.")
+
+    def extract(self):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+        response = requests.get(
+            url=self.url,
+            params=self.params,
+            headers=self.headers,
+            auth=self.auth,
+            stream=True,
+        )
+        response.raise_for_status()
+        logging.info(f"Successfully downloaded file from {response.url}")
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data
+
+    @staticmethod
+    def format_url(url_template, url_variables):
+        try:
+            url = url_template.format(**url_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing configuration url_variable: {missing_key}")
+
+        return url

hydroserverpy/etl/extractors/local_file_extractor.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Dict
+
+from .base import Extractor
+from ..types import TimeRange
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, filepath: str):
+        self.filepath = filepath
+
+    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
+        pass
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.filepath, "r")
+            logging.info(f"Successfully opened file '{self.filepath}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.filepath}': {e}")
+            return None