gensor 0.0.4__tar.gz → 0.0.5__tar.gz

{gensor-0.0.4 → gensor-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gensor
-Version: 0.0.4
+Version: 0.0.5
 Summary: Library for handling groundwater sensor data.
 Home-page: https://github.com/zawadzkim/gensor
 Author: Mateusz Zawadzki
@@ -15,7 +15,7 @@ Requires-Dist: numpy (>=2.1.0,<3.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pandera (>=0.20.3,<0.21.0)
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
-Requires-Dist: pytz (>=2024.1,<2025.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: scikit-learn (>=1.5.1,<2.0.0)
 Requires-Dist: scipy (>=1.14.1,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.32,<3.0.0)

{gensor-0.0.4 → gensor-0.0.5}/gensor/__init__.py

@@ -1,14 +1,18 @@
 from .compensation import Compensator, compensate
 from .dtypes import Dataset, Timeseries
-from .getters import read_from_csv
+from .getters import read_from_csv, read_from_sql
 from .preprocessing import OutlierDetection, Transform
 
 __all__ = [
+    # basic data types
     "Dataset",
     "Timeseries",
-    "read_from_csv",
+    # data transformation
     "OutlierDetection",
     "Transform",
     "Compensator",
     "compensate",
+    # getters
+    "read_from_csv",
+    "read_from_sql",
 ]

gensor-0.0.5/gensor/db/connection.py

@@ -0,0 +1,106 @@
+"""Module defining database connection object.
+
+Classes:
+    DatabaseConnection: Database connection object
+"""
+
+from pathlib import Path
+from typing import Any
+
+import pydantic as pyd
+from sqlalchemy import (
+    Column,
+    Connection,
+    Engine,
+    Float,
+    MetaData,
+    String,
+    Table,
+    create_engine,
+)
+
+from ..exceptions import DatabaseNotFound
+
+
+class DatabaseConnection(pyd.BaseModel):
+    """Database connection object.
+    If no database exists at the specified path, it will be created.
+    If no database is specified, an in-memory database will be used."""
+
+    model_config = pyd.ConfigDict(
+        arbitrary_types_allowed=True, validate_assignment=True
+    )
+
+    metadata: MetaData = MetaData()
+    db_directory: Path = Path.cwd()
+    db_name: str = "gensor.db"
+    engine: Engine | None = None
+
+    def _verify_path(self) -> str:
+        """Verify database path."""
+
+        if not self.db_directory.exists():
+            raise DatabaseNotFound()
+        return f"sqlite:///{self.db_directory}/{self.db_name}"
+
+    def connect(self) -> Connection:
+        """Connect to the database and initialize the engine.
+        If engine is None > create it with verified path > reflect
+        """
+        if self.engine is None:
+            sqlite_path = self._verify_path()
+            self.engine = create_engine(sqlite_path)
+        return self.engine.connect()
+
+    def dispose(self) -> None:
+        """Dispose of the engine, closing all connections."""
+        if self.metadata:
+            self.metadata.clear()
+        if self.engine:
+            self.engine.dispose()
+
+    def __enter__(self) -> Connection:
+        """Enable usage in a `with` block by returning the engine."""
+        con = self.connect()
+        if self.engine:
+            self.metadata.reflect(bind=self.engine)
+        return con
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Dispose of the engine when exiting the `with` block."""
+        self.dispose()
+
+    def get_tables(self) -> list | None:
+        with self:
+            tables = self.metadata.tables
+
+            if not tables:
+                print("This database has no tables.")
+                return None
+            else:
+                return list(tables)
+
+    def create_table(self, schema_name: str, column_name: str) -> Table | str:
+        """Create a table in the database.
+
+        Schema name is a string representing the location, sensor, variable measured and
+        unit of measurement. This is a way of preserving the metadata of the Timeseries.
+        The index is always `timestamp` and the column name is dynamicly create from
+        the measured variable.
+        """
+
+        if schema_name in self.metadata.tables:
+            return self.metadata.tables[schema_name]
+
+        ts_table = Table(
+            schema_name,
+            self.metadata,
+            Column("timestamp", String, primary_key=True),
+            Column(column_name, Float),
+        )
+        if self.engine:
+            ts_table.create(self.engine, checkfirst=True)
+            self.metadata.reflect(bind=self.engine)
+            return ts_table
+        else:
+            return "Engine does not exist."

{gensor-0.0.4 → gensor-0.0.5}/gensor/dtypes.py

@@ -1,12 +1,24 @@
+"""
+!!! warning
+
+    Whenever Timeseries objects are created via read_from_csv and use a parser (e.g.,
+    'vanessen'), the timestamps are localized and converted to UTC. Therefore, if the
+    user creates his own timeseries outside the read_from_csv, they should ensure that
+    the timestamps are in UTC format.
+"""
+
 from __future__ import annotations
 
+from collections import defaultdict
 from collections.abc import Callable
-from typing import Any, Literal
+from typing import Any, Literal, Self
 
 import pandas as pd
 import pandera as pa
 import pydantic as pyd
 from matplotlib import pyplot as plt
+from sqlalchemy import Table
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
 
 from .db import DatabaseConnection
 from .exceptions import IndexOutOfRangeError, TimeseriesNotFound, TimeseriesUnequal
@@ -14,14 +26,14 @@ from .preprocessing import OutlierDetection, Transform
 
 ts_schema = pa.SeriesSchema(
     float,
-    index=pa.Index(pa.DateTime, coerce=True),
+    index=pa.Index(pd.DatetimeTZDtype(tz="UTC"), coerce=False),
     coerce=True,
 )
 
 VARIABLE_TYPES_AND_UNITS = {
-    "temperature": ["degC"],
-    "pressure": ["cmH2O", "mmH2O"],
-    "conductivity": ["mS/cm"],
+    "temperature": ["degc"],
+    "pressure": ["cmh2o", "mmh2o"],
+    "conductivity": ["ms/cm"],
     "flux": ["m/s"],
     "head": ["m asl"],
     "depth": ["m"],
@@ -63,7 +75,7 @@ class Timeseries(pyd.BaseModel):
     variable: Literal[
         "temperature", "pressure", "conductivity", "flux", "head", "depth"
     ]
-    unit: Literal["degC", "cmH2O", "mS/cm", "m/s", "m asl", "m"]
+    unit: Literal["degc", "cmh2o", "ms/cm", "m/s", "m asl", "m"]
     location: str | None = None
     sensor: str | None = None
     sensor_alt: float | None = None
@@ -213,25 +225,41 @@ class Timeseries(pyd.BaseModel):
     def to_sql(self, db: DatabaseConnection) -> str:
         """Converts the timeseries to a list of dictionaries and uploads it to the database.
 
-        Normally the upload of the data with SQLAlchemy ORM would require creation of LoggerRecords instances,
-        but since the on_conflict_do_nothing clause is is used to avoid inserting duplicate rows, the
-        data has to be uploaded as a list of dictionaries.
+        The Timeseries data is uploaded to the SQL database by using the pandas
+        `to_sql` method.
 
         Args:
-            db (DatabaseConnection): The database connection object (see gwlogger.db.connection).
+            db (DatabaseConnection): The database connection object.
 
         Returns:
             str: A message indicating the number of rows inserted into the database.
         """
-        schema_name = f"{self.location}_{self.sensor}_{self.variable}_{self.unit}"
-        if db.engine is not None:
-            with db.engine.connect() as con:
-                self.ts.to_sql(
-                    name=schema_name, con=con, if_exists="append", index=False
-                )
+        schema_name = (
+            f"{self.location}_{self.sensor}_{self.variable}_{self.unit}".lower()
+        )
+
+        if isinstance(self.ts.index, pd.DatetimeIndex):
+            utc_index = (
+                self.ts.index.tz_convert("UTC")
+                if self.ts.index.tz is not None  # tzinfo becomes tz for DatetimeIndex
+                else self.ts.index
+            )
         else:
-            message = "Database engine is not initialized."
-            raise ValueError(message)
+            message = "The index is not a DatetimeIndex and cannot be converted to UTC."
+            raise TypeError(message)
+
+        series_as_records = list(
+            zip(utc_index.strftime("%Y-%m-%dT%H:%M:%S%z"), self.ts, strict=False)
+        )
+
+        with db as con:
+            schema = db.create_table(schema_name, self.variable)
+            if isinstance(schema, Table):
+                stmt = sqlite_insert(schema).values(series_as_records)
+                stmt = stmt.on_conflict_do_nothing(index_elements=["timestamp"])
+
+                con.execute(stmt)
+                con.commit()
 
         return f"{schema_name} table updated."
 
@@ -258,7 +286,7 @@ class Timeseries(pyd.BaseModel):
         ax.plot(
             self.ts.index,
             self.ts,
-            label=f"{self.variable} ({self.unit})",
+            label=f"{self.location} ({self.sensor})",
             **plot_kwargs,
         )
 
@@ -329,7 +357,7 @@ class Dataset(pyd.BaseModel):
         """List all unique locations in the dataset."""
         return [ts.location for ts in self.timeseries if ts is not None]
 
-    def add(self, other: Timeseries | list[Timeseries]) -> None:
+    def add(self, other: Timeseries | list[Timeseries] | Self) -> None:
         """Appends a new series to the Dataset or merges series if an equal
         one exists.
 
@@ -342,8 +370,13 @@ class Dataset(pyd.BaseModel):
         """
         if isinstance(other, list):
             for ts in other:
-                self._add_single_timeseries(ts)
-        else:
+                if isinstance(ts, Timeseries):
+                    self._add_single_timeseries(ts)
+        elif isinstance(other, Dataset):
+            for ts in other.timeseries:  # type: ignore[assignment]
+                if isinstance(ts, Timeseries):
+                    self._add_single_timeseries(ts)
+        elif isinstance(other, Timeseries):
             self._add_single_timeseries(other)
 
         return
@@ -395,9 +428,38 @@ class Dataset(pyd.BaseModel):
 
         return self.model_copy(update={"timeseries": matching_timeseries})
 
+    def to_sql(self, db: DatabaseConnection) -> None:
+        for ts in self.timeseries:
+            if ts:
+                ts.to_sql(db)
+        return
+
+    def plot(self, include_outliers: bool = False) -> None:
+        """Plots the timeseries data, grouping by variable type.
+
+        Args:
+            include_outliers (bool): Whether to include outliers in the plot.
+        """
+        # Group timeseries by variable
+        grouped_ts = defaultdict(list)
+        for ts in self.timeseries:
+            if ts:
+                grouped_ts[ts.variable].append(ts)
+
+        # Create a plot for each group of timeseries with the same variable
+        for variable, ts_list in grouped_ts.items():
+            fig, ax = plt.subplots(figsize=(10, 5))
+            for ts in ts_list:
+                ts.plot(include_outliers=include_outliers, ax=ax)
+
+            ax.set_title(f"Timeseries for {variable.capitalize()}")
+            plt.show()
+
+        return
+
     # def align(self,
-    #           freq: str = 'h',
-    #           inplace: bool = True):
+    #             freq: str = 'h',
+    #             inplace: bool = True):
     #     """Aligns the timeseries to a common time axis.
 
     #     Args:
@@ -406,7 +468,7 @@ class Dataset(pyd.BaseModel):
     #     """
 
     #     index_sets = [set(serie._resample(freq).index)
-    #                   for serie in self.timeseries]
+    #                     for serie in self.timeseries]
 
     #     # Find the intersection of all index sets to get the common dates
     #     common_dates = set.intersection(*index_sets)
@@ -430,21 +492,3 @@ class Dataset(pyd.BaseModel):
     #         aligned_series = Dataset(aligned_series)
 
     #     return aligned_series
-
-
-#     def plot(self, stations: list[str] | None = None):
-#         """Plots the timeseries data.
-
-#         Args:
-#             ts (Timeseries): The timeseries to plot.
-#         """
-#         plt.figure(figsize=(10, 5))
-
-#         for ts in self.timeseries:
-#             plt.plot(ts.timeseries.index, ts.timeseries,
-#                      label=f'{ts.measurement_type} at {ts.station}')
-#         plt.xlabel('Time')
-#         plt.ylabel('Value')
-#         plt.title('Timeseries data')
-#         plt.legend()
-#         plt.show()

gensor-0.0.5/gensor/getters.py

@@ -0,0 +1,131 @@
+"""Fetching the data from various sources.
+
+TODO: Fix up the read_from_sql() function to actually work properly.
+"""
+
+from pathlib import Path
+from typing import Any, Literal
+
+import pandas as pd
+from sqlalchemy import select
+
+from .db.connection import DatabaseConnection
+from .dtypes import Dataset, Timeseries
+from .exceptions import NoFilesToLoad
+from .parse import parse_vanessen_csv
+
+
+def read_from_csv(
+    path: Path, file_format: Literal["vanessen"] = "vanessen", **kwargs: Any
+) -> Dataset:
+    """Loads the data from the Van Essen CSV file(s) and returns a list of Timeseries objects.
+
+    Args:
+        path (Path): The path to the file or directory containing the files.
+        **kwargs (dict): Optional keyword arguments passed to `parse_vanessen_csv()` to specify the regex patterns for the serial number and station.
+            serial_number_pattern (str): The regex pattern to extract the serial number from the file.
+            location_pattern (str): The regex pattern to extract the station from the file.
+            col_names (list): The column names for the dataframe.
+    """
+
+    parsers = {
+        "vanessen": parse_vanessen_csv,
+    }
+
+    if not isinstance(path, Path):
+        message = "The path argument must be a Path object."
+        raise TypeError(message)
+
+    if path.is_dir() and not any(path.iterdir()):
+        raise NoFilesToLoad()
+
+    files = (
+        [file for file in path.iterdir() if file.is_file()] if path.is_dir() else [path]
+    )
+
+    parser = parsers[file_format]
+    ds = Dataset()
+    for f in files:
+        print(f"Loading file: {f}")
+        ts_in_file = parser(f, **kwargs)
+        ds.add(ts_in_file)
+
+    return ds
+
+
+def read_from_sql(
+    db: DatabaseConnection,
+    load_all: bool,
+    location: str | None = None,
+    sensor: str | None = None,
+    variable: str | None = None,
+    unit: str | None = None,
+) -> Timeseries | Dataset:
+    """Returns the timeseries or a dataset from a SQL database.
+
+    Parameters:
+        db (DatabaseConnection): The database connection object.
+        load_all (bool): Whether to load all timeseries from the database.
+        location (str): The station name.
+        sensor (str): The sensor name.
+        variable (str): The measurement type.
+        unit (str): The unit of the measurement.
+
+    Returns:
+        Timeseries: The Timeseries object retrieved from the database.
+
+    Raises:
+        ValueError: If the DataFrame cannot be retrieved or if it's empty.
+        TypeError: If the retrieved data is not a DataFrame or is of incorrect type.
+    """
+
+    def _read_from_sql(
+        location: str, sensor: str, variable: str, unit: str
+    ) -> Timeseries:
+        schema_name = f"{location}_{sensor}_{variable}_{unit}".lower()
+
+        with db as con:
+            schema = db.metadata.tables[schema_name]
+            query = select(schema)
+            ts = pd.read_sql(
+                query,
+                con=con,
+                parse_dates={"timestamp": "%Y-%m-%dT%H:%M:%S%z"},
+                index_col="timestamp",
+            ).squeeze()
+        if ts.empty:
+            message = f"No data found in table {schema_name}"
+            raise ValueError(message)
+
+        # Variable and type validation are handled by pydantic model
+        ts_object = Timeseries(
+            ts=ts,
+            variable=variable,  # type: ignore[arg-type]
+            location=location,
+            sensor=sensor,
+            unit=unit,  # type: ignore[arg-type]
+        )
+
+        return ts_object
+
+    # fmt: off
+    if load_all:
+        schemas = db.get_tables()
+        if schemas:
+            timeseries = [_read_from_sql(*ts_name.split("_"))
+                          for ts_name in schemas]
+
+            return Dataset(timeseries=[ts for ts in timeseries if ts is not None])
+        else:
+            return Dataset()
+    else:
+
+        return _read_from_sql(location, sensor, variable, unit)  # type: ignore[arg-type]
+
+
+# fmt: on
+
+
+def read_from_api() -> Dataset:
+    """Fetch data from the API."""
+    return NotImplemented

{gensor-0.0.4 → gensor-0.0.5}/gensor/parse/vanessen.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from typing import Any
 
 import chardet
-import pytz
+from dateutil import tz
 from pandas import DataFrame, read_csv, to_datetime
 
 from ..dtypes import VARIABLE_TYPES_AND_UNITS, Timeseries
@@ -28,25 +28,20 @@ def detect_encoding(path: Path, num_bytes: int = 1024) -> str:
     return result["encoding"] or "utf-8"
 
 
-def handle_timestamps(df: DataFrame, tz: str) -> DataFrame:
-    """Converts the timestamps in the dataframe to the specified timezone.
-
-    The timezone is obtained from the file metadata. If the timezone is UTC, the offset is extracted
-    and the timestamps are converted to the corresponding timezone. If the timezone is not UTC, the
-    timestamps are converted to UTC and then to the specified timezone.
+def handle_timestamps(df: DataFrame, tz_string: str) -> DataFrame:
+    """Converts timestamps in the dataframe to the specified timezone (e.g., 'UTC+1').
 
     Args:
-        df (pd.DataFrame): The dataframe with the data.
-        tz (str): The timezone string obtained from the file metadata.
-    """
+        df (pd.DataFrame): The dataframe with timestamps.
+        tz_string (str): A timezone string like 'UTC+1' or 'UTC-5'.
 
-    if tz.startswith("UTC"):
-        offset_hours = int(tz[3:])
-        timezone = pytz.FixedOffset(offset_hours * 60)
-    else:
-        timezone = pytz.UTC
+    Returns:
+        pd.DataFrame: The dataframe with timestamps converted to UTC.
+    """
+    timezone = tz.gettz(tz_string)
 
-    df.index = to_datetime(df.index).tz_localize("UTC").tz_convert(timezone)
+    df.index = to_datetime(df.index).tz_localize(timezone)
+    df.index = df.index.tz_convert("UTC")
 
     return df
 

{gensor-0.0.4 → gensor-0.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gensor"
-version = "0.0.4"
+version = "0.0.5"
 description = "Library for handling groundwater sensor data."
 authors = ["Mateusz Zawadzki <zawadzkimat@outlook.com>"]
 repository = "https://github.com/zawadzkim/gensor"
@@ -22,7 +22,7 @@ numpy = "^2.1.0"
 scikit-learn = "^1.5.1"
 sqlalchemy = "^2.0.32"
 pandas = "^2.2.2"
-pytz = "^2024.1"
+python-dateutil = "^2.9.0.post0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.0"

gensor-0.0.4/gensor/db/connection.py

@@ -1,53 +0,0 @@
-"""Module defining database connection object.
-
-Classes:
-    DatabaseConnection: Database connection object
-"""
-
-from pathlib import Path
-
-import pydantic as pyd
-from sqlalchemy import Engine, create_engine
-from sqlalchemy.orm import Session, sessionmaker
-
-from ..exceptions import DatabaseNotFound
-
-
-class DatabaseConnection(pyd.BaseModel):
-    """Database connection object.
-    If no database exists at the specified path, it will be created.
-    If no database is specified, an in-memory database will be used.
-
-    The user should specify the database directory and name separately. If directory is not specified,
-    current directory and a default name are used. ."""
-
-    model_config = pyd.ConfigDict(
-        arbitrary_types_allowed=True, validate_assignment=True
-    )
-
-    in_memory: bool = False
-    db_directory: Path = Path.cwd()
-    db_name: str = "gensor.db"
-    engine: Engine | None = None
-    session: Session | None = None
-
-    def __post_init__(self) -> None:
-        self.connect()
-
-    def _verify_path(self) -> str:
-        if self.in_memory:
-            return "sqlite:///:memory:"
-        else:
-            if not self.db_directory.exists():
-                raise DatabaseNotFound()
-            else:
-                return f"sqlite:///{self.db_directory}/{self.db_name}"
-
-    def connect(self) -> Session:
-        sqlite_path = self._verify_path()
-
-        self.engine = create_engine(sqlite_path)
-        session = sessionmaker(bind=self.engine)
-        self.session = session()
-
-        return session()

gensor-0.0.4/gensor/getters.py

@@ -1,95 +0,0 @@
-"""Fetching the data from various sources.
-
-TODO: Fix up the read_from_sql() function to actually work properly.
-"""
-
-from pathlib import Path
-from typing import Any, Literal
-
-from pandas import Series, read_sql
-from sqlalchemy import MetaData, Table, select
-
-from .db.connection import DatabaseConnection
-from .dtypes import Dataset, Timeseries
-from .exceptions import NoFilesToLoad
-from .parse import parse_vanessen_csv
-
-
-def read_from_csv(
-    path: Path, file_format: Literal["vanessen"] = "vanessen", **kwargs: Any
-) -> Dataset:
-    """Loads the data from the Van Essen CSV file(s) and returns a list of Timeseries objects.
-
-    Args:
-        path (Path): The path to the file or directory containing the files.
-        **kwargs (dict): Optional keyword arguments passed to `parse_vanessen_csv()` to specify the regex patterns for the serial number and station.
-            serial_number_pattern (str): The regex pattern to extract the serial number from the file.
-            location_pattern (str): The regex pattern to extract the station from the file.
-            col_names (list): The column names for the dataframe.
-    """
-
-    parsers = {
-        "vanessen": parse_vanessen_csv,
-    }
-
-    if not isinstance(path, Path):
-        message = "The path argument must be a Path object."
-        raise TypeError(message)
-
-    if path.is_dir() and not any(path.iterdir()):
-        raise NoFilesToLoad()
-
-    files = (
-        [file for file in path.iterdir() if file.is_file()] if path.is_dir() else [path]
-    )
-
-    parser = parsers[file_format]
-    ds = Dataset()
-    for f in files:
-        print(f"Loading file: {f}")
-        ts_in_file = parser(f, **kwargs)
-        ds.add(ts_in_file)
-
-    return ds
-
-
-def read_from_sql(
-    db: DatabaseConnection, location: str, sensor: str, variable: str, unit: str
-) -> list[Timeseries]:
-    """Returns the timeseries from a sql database.
-
-    Parameters:
-        db (DatabaseConnection): The database connection object
-        location (str): The station name
-        sensor (str): Sensor name
-        variable (str): The measurement type
-        unit (str): Unit of the measurement
-
-    """
-    metadata = MetaData()
-    schema = Table(f"{location}_{sensor}_{variable}", metadata)
-
-    query = select(schema)
-    if db.engine:
-        with db.engine.connect() as con:
-            df = read_sql(query, con=con, index_col="timestamp")
-
-    if not isinstance(df, Series):
-        raise TypeError
-
-    ts_object = Timeseries(
-        ts=df,
-        # Validation done in Pydantic
-        variable=variable,
-        location=location,
-        sensor=sensor,
-        # Validation done in Pydantic
-        unit=unit,
-    )
-
-    return ts_object
-
-
-def read_from_api() -> Dataset:
-    """Fetch data from the API."""
-    return NotImplemented