gensor 0.0.4__tar.gz → 0.0.6__tar.gz

{gensor-0.0.4 → gensor-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gensor
-Version: 0.0.4
+Version: 0.0.6
 Summary: Library for handling groundwater sensor data.
 Home-page: https://github.com/zawadzkim/gensor
 Author: Mateusz Zawadzki
@@ -11,14 +11,17 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: matplotlib (>=3.9.2,<4.0.0)
-Requires-Dist: numpy (>=2.1.0,<3.0.0)
+Requires-Dist: numba (>=0.60.0,<0.61.0)
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pandera (>=0.20.3,<0.21.0)
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
-Requires-Dist: pytz (>=2024.1,<2025.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: scikit-learn (>=1.5.1,<2.0.0)
 Requires-Dist: scipy (>=1.14.1,<2.0.0)
+Requires-Dist: seaborn (>=0.13.2,<0.14.0)
 Requires-Dist: sqlalchemy (>=2.0.32,<3.0.0)
+Requires-Dist: statsmodels (>=0.14.3,<0.15.0)
 Project-URL: Documentation, https://zawadzkim.github.io/gensor/
 Project-URL: Repository, https://github.com/zawadzkim/gensor
 Description-Content-Type: text/markdown

{gensor-0.0.4 → gensor-0.0.6}/gensor/__init__.py

@@ -1,14 +1,18 @@
 from .compensation import Compensator, compensate
 from .dtypes import Dataset, Timeseries
-from .getters import read_from_csv
+from .getters import read_from_csv, read_from_sql
 from .preprocessing import OutlierDetection, Transform
 
 __all__ = [
+    # basic data types
     "Dataset",
     "Timeseries",
-    "read_from_csv",
+    # data transformation
     "OutlierDetection",
     "Transform",
     "Compensator",
     "compensate",
+    # getters
+    "read_from_csv",
+    "read_from_sql",
 ]

gensor-0.0.6/gensor/compensation.py

@@ -0,0 +1,194 @@
+"""Compensating the raw data from the absolute pressure transducer to the actual water
+level using the barometric pressure data.
+
+Because van Essen Instrument divers are non-vented pressure transducers, to obtain the
+pressure resulting from the water column above the logger (i.e. the water level), the
+barometric pressure must be subtracted from the raw pressure measurements. In the
+first step the function aligns the two series to the same time step and then subtracts
+the barometric pressure from the raw pressure measurements. For short time periods (when
+for instance a slug test is performed) the barometric pressure can be provided as a
+single float value.
+
+Subsequently the function filters out all records where the absolute water column is
+less than or equal to the cutoff value. This is because when the logger is out of the
+water when the measurement is taken, the absolute water column is close to zero,
+producing erroneous results and spikes in the plots. The cutoff value is set to 5 cm by
+default, but can be adjusted using the cutoff_wc kwarg.
+
+Functions:
+
+    compensate: Compensate raw sensor pressure measurement with barometric pressure.
+"""
+
+from typing import Literal
+
+import pandas as pd
+import pydantic as pyd
+
+from .dtypes import Dataset, Timeseries
+from .exceptions import (
+    InvalidMeasurementTypeError,
+    MissingInputError,
+)
+
+
+class Compensator(pyd.BaseModel):
+    """Compensate raw sensor pressure measurement with barometric pressure.
+
+    Attributes:
+        ts (Timeseries): Raw sensor timeseries
+        barometric (Timeseries | float): Barometric pressure timeseries or a single
+            float value. If a float value is provided, it is assumed to be in cmH2O.
+    """
+
+    ts: Timeseries
+    barometric: Timeseries | float
+
+    @pyd.field_validator("ts", "barometric", mode="before")
+    def validate_timeseries_type(cls, v: Timeseries) -> Timeseries:
+        if isinstance(v, Timeseries) and v.variable != "pressure":
+            raise InvalidMeasurementTypeError()
+        return v
+
+    @pyd.field_validator("ts")
+    def validate_sensor_information(cls, v: Timeseries) -> Timeseries:
+        if v.sensor is not None and not v.sensor_alt:
+            raise MissingInputError("sensor_alt")
+        return v
+
+    def compensate(
+        self,
+        alignment_period: Literal["D", "ME", "SME", "MS", "YE", "YS", "h", "min", "s"],
+        threshold_wc: float | None,
+        fieldwork_dates: list | None,
+    ) -> Timeseries | None:
+        """Perform compensation.
+
+        Parameters:
+            alignment_period Literal['D', 'ME', 'SME', 'MS', 'YE', 'YS', 'h', 'min', 's']: The alignment period for the timeseries.
+                Default is 'h'. See pandas offset aliases for definitinos.
+            threshold_wc (float): The threshold for the absolute water column.
+            fieldwork_dates (Optional[list]): List of dates when fieldwork was done. All
+                measurement from a fieldwork day will be set to None.
+
+        Returns:
+            Timeseries: A new Timeseries instance with the compensated data and updated unit and variable. Optionally removed outliers are included.
+        """
+
+        resample_params = {"freq": alignment_period, "agg_func": pd.Series.mean}
+        resampled_ts = self.ts.resample(**resample_params)
+
+        if isinstance(self.barometric, Timeseries):
+            if self.ts == self.barometric:
+                print("Skipping compensation: both timeseries are the same.")
+                return None
+            resampled_baro = self.barometric.resample(**resample_params).ts
+
+        elif isinstance(self.barometric, float):
+            resampled_baro = pd.Series(
+                [self.barometric] * len(resampled_ts.ts), index=resampled_ts.ts.index
+            )
+
+        # dividing by 100 to convert water column from cmH2O to mH2O
+        watercolumn_ts = resampled_ts.ts.sub(resampled_baro).divide(100).dropna()
+
+        if not isinstance(watercolumn_ts.index, pd.DatetimeIndex):
+            watercolumn_ts.index = pd.to_datetime(watercolumn_ts.index)
+
+        if fieldwork_dates:
+            fieldwork_timestamps = pd.to_datetime(fieldwork_dates).tz_localize(
+                watercolumn_ts.index.tz
+            )
+
+            watercolumn_ts.loc[
+                watercolumn_ts.index.normalize().isin(fieldwork_timestamps)
+            ] = None
+
+        if threshold_wc:
+            watercolumn_ts_filtered = watercolumn_ts[
+                watercolumn_ts.abs() > threshold_wc
+            ]
+
+            dropped_outliers = watercolumn_ts[watercolumn_ts.abs() <= threshold_wc]
+
+            print(
+                f"{len(dropped_outliers)} records \
+                    dropped due to low water column."
+            )
+            gwl = watercolumn_ts_filtered.add(float(resampled_ts.sensor_alt or 0))
+
+            compensated = resampled_ts.model_copy(
+                update={
+                    "ts": gwl,
+                    "outliers": dropped_outliers,
+                    "unit": "m asl",
+                    "variable": "head",
+                },
+                deep=True,
+            )
+        else:
+            gwl = watercolumn_ts.add(float(resampled_ts.sensor_alt or 0))
+
+            compensated = resampled_ts.model_copy(
+                update={"ts": gwl, "unit": "m asl", "variable": "head"}, deep=True
+            )
+
+        return compensated
+
+
+def compensate(
+    raw: Timeseries | Dataset,
+    barometric: Timeseries | float,
+    alignment_period: Literal[
+        "D", "ME", "SME", "MS", "YE", "YS", "h", "min", "s"
+    ] = "h",
+    threshold_wc: float | None = None,
+    fieldwork_dates: dict | None = None,
+    interpolate_method: str | None = None,
+) -> Timeseries | Dataset | None:
+    """Constructor for the Comensator object.
+
+    Parameters:
+        raw (Timeseries | Dataset): Raw sensor timeseries
+        barometric (Timeseries | float): Barometric pressure timeseries or a single
+            float value. If a float value is provided, it is assumed to be in cmH2O.
+        alignment_period (Literal['D', 'ME', 'SME', 'MS', 'YE', 'YS', 'h', 'min', 's']): The alignment period for the timeseries.
+            Default is 'h'. See pandas offset aliases for definitinos.
+        threshold_wc (float): The threshold for the absolute water column. If it is
+            provided, the records below that threshold are dropped.
+        fieldwork_dates (Dict[str, list]): Dictionary of location name and a list of
+            fieldwork days. All records on the fieldwork day are set to None.
+        interpolate_method (str): String representing the interpolate method as in
+            pd.Series.interpolate() method.
+    """
+    if fieldwork_dates is None:
+        fieldwork_dates = {}
+
+    def _compensate_one(
+        raw: Timeseries, fieldwork_dates: list | None
+    ) -> Timeseries | None:
+        comp = Compensator(ts=raw, barometric=barometric)
+        compensated = comp.compensate(
+            alignment_period=alignment_period,
+            threshold_wc=threshold_wc,
+            fieldwork_dates=fieldwork_dates,
+        )
+        if compensated is not None and interpolate_method:
+            # .interpolate() called on Timeseries object is wrapped to return a
+            # Timeseries object from the original pandas.Series.interpolate().
+            return compensated.interpolate(method=interpolate_method)  # type: ignore[no-any-return]
+
+        else:
+            return compensated
+
+    if isinstance(raw, Timeseries):
+        dates = fieldwork_dates.get(raw.location)
+        return _compensate_one(raw, dates)
+
+    elif isinstance(raw, Dataset):
+        compensated_series = []
+        for item in raw:
+            dates = fieldwork_dates.get(item.location)
+            compensated_series.append(_compensate_one(item, dates))
+
+        return raw.model_copy(update={"timeseries": compensated_series}, deep=True)

gensor-0.0.6/gensor/db/connection.py

@@ -0,0 +1,144 @@
+"""Module defining database connection object.
+
+Classes:
+    DatabaseConnection: Database connection object
+"""
+
+from pathlib import Path
+from typing import Any
+
+import pydantic as pyd
+from sqlalchemy import (
+    Column,
+    Connection,
+    Engine,
+    Float,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    create_engine,
+)
+
+from ..exceptions import DatabaseNotFound
+
+
+class DatabaseConnection(pyd.BaseModel):
+    """Database connection object.
+    If no database exists at the specified path, it will be created.
+    If no database is specified, an in-memory database will be used."""
+
+    model_config = pyd.ConfigDict(
+        arbitrary_types_allowed=True, validate_assignment=True
+    )
+
+    metadata: MetaData = MetaData()
+    db_directory: Path = Path.cwd()
+    db_name: str = "gensor.db"
+    engine: Engine | None = None
+
+    def _verify_path(self) -> str:
+        """Verify database path."""
+
+        if not self.db_directory.exists():
+            raise DatabaseNotFound()
+        return f"sqlite:///{self.db_directory}/{self.db_name}"
+
+    def connect(self) -> Connection:
+        """Connect to the database and initialize the engine.
+        If engine is None > create it with verified path > reflect.
+        After connecting, ensure the timeseries_metadata table is present.
+        """
+        if self.engine is None:
+            sqlite_path = self._verify_path()
+            self.engine = create_engine(sqlite_path)
+
+        connection = self.engine.connect()
+
+        self.create_metadata()
+
+        return connection
+
+    def dispose(self) -> None:
+        """Dispose of the engine, closing all connections."""
+        if self.metadata:
+            self.metadata.clear()
+        if self.engine:
+            self.engine.dispose()
+
+    def __enter__(self) -> Connection:
+        """Enable usage in a `with` block by returning the engine."""
+        con = self.connect()
+        if self.engine:
+            self.metadata.reflect(bind=self.engine)
+        return con
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Dispose of the engine when exiting the `with` block."""
+        self.dispose()
+
+    def get_tables(self) -> list | None:
+        """Return the list of tables, excluding the 'timeseries_metadata' table."""
+        with self:
+            tables = self.metadata.tables
+
+            if not tables:
+                print("This database has no tables.")
+                return None
+            else:
+                filtered_tables = [
+                    table for table in tables if table != "__timeseries_metadata__"
+                ]
+                return filtered_tables
+
+    def create_metadata(self) -> str | Table:
+        """Create a metadata table if it doesn't exist yet and store ts metadata."""
+
+        metadata_table = Table(
+            "__timeseries_metadata__",
+            self.metadata,
+            Column("id", Integer, primary_key=True),
+            Column("table_name", String, unique=True),
+            Column("location", String),
+            Column("sensor", String),
+            Column("variable", String),
+            Column("unit", String),
+            Column("logger_alt", Float, nullable=True),
+            Column("location_alt", Float, nullable=True),
+            Column("timestamp_start", String, nullable=True),
+            Column("timestamp_end", String, nullable=True),
+        )
+
+        if self.engine:
+            metadata_table.create(self.engine, checkfirst=True)
+            self.metadata.reflect(bind=self.engine)
+            return metadata_table
+        else:
+            return "Engine does not exist."
+
+    def create_table(self, schema_name: str, column_name: str) -> Table | str:
+        """Create a table in the database.
+
+        Schema name is a string representing the location, sensor, variable measured and
+        unit of measurement. This is a way of preserving the metadata of the Timeseries.
+        The index is always `timestamp` and the column name is dynamicly create from
+        the measured variable.
+        """
+
+        if schema_name in self.metadata.tables:
+            return self.metadata.tables[schema_name]
+
+        ts_table = Table(
+            schema_name,
+            self.metadata,
+            Column("timestamp", String, primary_key=True),
+            Column(column_name, Float),
+            info={},
+        )
+
+        if self.engine:
+            ts_table.create(self.engine, checkfirst=True)
+            self.metadata.reflect(bind=self.engine)
+            return ts_table
+        else:
+            return "Engine does not exist."