teehr 0.4.13__tar.gz → 0.5.1.dev10__tar.gz

{teehr-0.4.13 → teehr-0.5.1.dev10}/PKG-INFO

@@ -1,42 +1,46 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: teehr
-Version: 0.4.13
+Version: 0.5.1.dev10
 Summary: Tools for Exploratory Evaluation in Hydrologic Research
 License: GNU v3
+License-File: LICENSE.md
 Author: RTI International
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.10,<3.14
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: arch (>=7.0.0,<8.0.0)
-Requires-Dist: bokeh (>=3.5.0,<4.0.0)
-Requires-Dist: dask[dataframe] (==2025.*)
-Requires-Dist: dataretrieval (>=1.0.9,<2.0.0)
-Requires-Dist: fsspec (>=2023.10.0)
-Requires-Dist: gcsfs (>=2023.10.0)
-Requires-Dist: geopandas (>=0.14.0,<0.15.0)
-Requires-Dist: geoviews (>=1.14.0,<2.0.0)
-Requires-Dist: h5py (==3.12.1)
-Requires-Dist: httpx (>=0.25.1,<0.26.0)
-Requires-Dist: hvplot (>=0.11.1,<0.12.0)
-Requires-Dist: kerchunk (>=0.2.2,<0.3.0)
-Requires-Dist: lxml (>=5.3.2,<6.0.0)
-Requires-Dist: netcdf4 (==1.6.5)
-Requires-Dist: numba (>=0.60.0,<0.61.0)
-Requires-Dist: pandas (>=2.2.0,<3.0.0)
-Requires-Dist: pandera[pyspark] (>=0.20.4,<0.21.0)
-Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
-Requires-Dist: pydantic (>=2.4.2,<3.0.0)
-Requires-Dist: pyspark[pandas-on-spark] (>=3.5.2,<4.0.0)
-Requires-Dist: rasterio (>=1.3.9,<2.0.0)
-Requires-Dist: rioxarray (>=0.15.0,<0.16.0)
-Requires-Dist: s3fs (>=2023.10.0)
-Requires-Dist: scoringrules (>=0.7.1,<0.8.0)
-Requires-Dist: ujson (>=5.8.0,<6.0.0)
-Requires-Dist: zarr (>=2.16.1,<3.0.0)
+Requires-Dist: apache-sedona (>=1.7.0,<2)
+Requires-Dist: arch (>=7.0.0,<8)
+Requires-Dist: baseflow (>=0.1.0,<0.2.0)
+Requires-Dist: bokeh (>=3.7.3,<4)
+Requires-Dist: dask (>=2025,<2026)
+Requires-Dist: dataretrieval (>=1.0.9,<2)
+Requires-Dist: fsspec (>=2023.10.0,<2026)
+Requires-Dist: gcsfs (>=2023.10.0,<2026)
+Requires-Dist: geopandas (>=1,<2)
+Requires-Dist: h5py (>=3.12.1,<4)
+Requires-Dist: httpx (>=0.25.1,<1)
+Requires-Dist: kerchunk (>=0.2.2,<1)
+Requires-Dist: lxml (>=5.4.0,<6)
+Requires-Dist: netcdf4 (>=1.6.5,<2)
+Requires-Dist: numba (>=0.60.0,<1)
+Requires-Dist: pandas (>=2.2.0,<3)
+Requires-Dist: pandera[pyspark] (>=0.20.4,<1)
+Requires-Dist: protobuf (==5.28.3)
+Requires-Dist: psutil (>=7.0.0,<8)
+Requires-Dist: pyarrow (>=15.0.0,<21)
+Requires-Dist: pydantic (>=2.4.2,<3)
+Requires-Dist: pyiceberg (>=0.9.1,<1)
+Requires-Dist: pyspark[pandas-on-spark] (>=4,<5)
+Requires-Dist: rasterio (>=1.3.0,<2)
+Requires-Dist: rioxarray (>=0.15.0,<1)
+Requires-Dist: s3fs (>=2023.10.0,<2026)
+Requires-Dist: scoringrules (>=0.7.1,<1)
+Requires-Dist: ujson (>=5.8.0,<6)
+Requires-Dist: zarr (>=2.16.1,<3)
 Description-Content-Type: text/markdown
 
 ![alt text](https://github.com/RTIInternational/teehr/blob/main/docs/images/teehr.png)

teehr-0.5.1.dev10/pyproject.toml

@@ -0,0 +1,91 @@
+[tool.poetry]
+name = "teehr"
+version = "0.5.1dev10"
+description = "Tools for Exploratory Evaluation in Hydrologic Research"
+authors = [
+    "RTI International",
+    "Matthew Denno <mdenno@rti.org>",
+    "Katie van Werkhoven <kvanwerkhoven@rti.org>",
+    "Sam Lamont <slamont@rti.org>",
+    "Sam Landsteiner <slandsteiner@rti.org>",
+]
+license = "GNU v3"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.14"
+geopandas = ">=1,<2"
+pydantic = ">=2.4.2,<3"
+dask = ">=2025,<2026"
+fsspec = ">=2023.10.0,<2026"
+ujson = ">=5.8.0,<6"
+kerchunk = ">=0.2.2,<1"
+gcsfs = ">=2023.10.0,<2026"
+s3fs = ">=2023.10.0,<2026"
+zarr = ">=2.16.1,<3"
+rioxarray = ">=0.15.0,<1"
+rasterio = ">=1.3.0,<2"
+h5py = ">=3.12.1,<4"
+pyarrow = ">=15.0.0,<21"
+httpx = ">=0.25.1,<1"
+pandas = ">=2.2.0,<3"
+pyspark = {extras = ["pandas-on-spark"], version = ">=4,<5"}
+dataretrieval = ">=1.0.9,<2"
+numba = ">=0.60.0,<1"
+arch = ">=7.0.0,<8"
+pandera = {extras = ["pyspark"], version = ">=0.20.4,<1"}
+netcdf4 = ">=1.6.5,<2"
+apache-sedona = ">=1.7.0,<2"
+scoringrules = ">=0.7.1,<1"
+pyiceberg = ">=0.9.1,<1"
+lxml = ">=5.4.0,<6"
+bokeh = ">=3.7.3,<4"
+protobuf = "5.28.3"
+psutil = ">=7.0.0,<8"
+baseflow = "^0.1.0"
+
+[tool.poetry.group.test.dependencies]
+pytest = ">=7.4.3"
+flake8 = ">=6.1.0"
+nox-poetry = "^1.2.0"
+pytest-cov = "^6.2.1"
+pytest-json-ctrf = "^0.3.5"
+
+[tool.poetry.group.dev.dependencies]
+sphinx = ">=7.2.6"
+sphinx-design = ">=0.5.0"
+numpydoc = ">=1.6.0"
+pre-commit = ">=3.6.0"
+pre-commit-hooks = ">=4.5.0"
+flake8-docstrings = ">=1.7.0"
+myst-nb = ">=1.0.0"
+nbstripout = ">=0.7.1"
+pickleshare = ">=0.7.5"
+sphinx-autoapi = "3.0.0"
+pydata-sphinx-theme = ">=0.16.0"
+sphinx-autobuild = ">=2024.10.3"
+dask = {extras = ["distributed"], version = "^2025.5.1"}
+tomli = ">=2.2.1"
+geoviews = ">=1.14.1,<2"
+hvplot = ">=0.12.0,<1"
+
+
+[tool.numpydoc_validation]
+checks = [
+    "all",   # report on all checks, except the below
+    "EX01",  # No examples section found
+    "RT01",  # No Returns section found
+    "SA01",  # See Also section not found
+    "ES01",  # No extended summary found
+    "GL01",  # Docstring text (summary) should start in the line immediately
+    "SS06",  # Summary should fit in a single line
+    "PR01",  # Parameters {missing_params} not documented
+    "GL02"   # Closing quotes should be placed in the line after the last text in docstring"
+]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"

{teehr-0.4.13 → teehr-0.5.1.dev10}/src/teehr/__init__.py

@@ -1,10 +1,16 @@
 """Initialize the TEEHR package."""
-__version__ = "0.4.13"
+import warnings
+
+__version__ = "0.5.1dev10"
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", UserWarning)
+    import pandera.pyspark as ps  # noqa: F401
 
 from teehr.evaluation.evaluation import Evaluation  # noqa
 from teehr.models.metrics.deterministic_models import DeterministicMetrics  # noqa
 from teehr.models.metrics.probabilistic_models import ProbabilisticMetrics  # noqa
-from teehr.models.metrics.signature_models import SignatureMetrics  # noqa
+from teehr.models.metrics.signature_models import Signatures  # noqa
 from teehr.models.metrics.bootstrap_models import Bootstrappers  # noqa
 from teehr.models.metrics.basemodels import Operators  # noqa
 from teehr.models.pydantic_table_models import (  # noqa
@@ -26,4 +32,8 @@ from teehr.models.filters import (  # noqa
 )
 
 from teehr.models.calculated_fields.row_level import RowLevelCalculatedFields # noqa
-from teehr.models.calculated_fields.timeseries_aware import TimeseriesAwareCalculatedFields # noqa
+from teehr.models.calculated_fields.timeseries_aware import TimeseriesAwareCalculatedFields # noqa
+from teehr.models.generate.timeseries_generator_models import (  # noqa
+    SignatureTimeseriesGenerators,
+    BenchmarkForecastGenerators
+)

{teehr-0.4.13 → teehr-0.5.1.dev10}/src/teehr/evaluation/evaluation.py

@@ -1,4 +1,5 @@
 """Evaluation module."""
+import psutil
 from datetime import datetime
 from typing import Union, Literal, List
 from pathlib import Path
@@ -22,11 +23,19 @@ from teehr.loading.s3.clone_from_s3 import (
     list_s3_evaluations,
     clone_from_s3
 )
+from teehr.models.filters import TableFilter, FilterBaseModel, TableNamesEnum
 import teehr.const as const
 from teehr.evaluation.fetch import Fetch
 from teehr.evaluation.metrics import Metrics
+from teehr.evaluation.generate import GeneratedTimeseries
 import pandas as pd
 from teehr.visualization.dataframe_accessor import TEEHRDataFrameAccessor # noqa
+import re
+import teehr
+import s3fs
+from fsspec.implementations.local import LocalFileSystem
+import pyspark.sql as ps
+from teehr.querying.filter_format import validate_and_apply_filters
 
 
 logger = logging.getLogger(__name__)
@@ -81,9 +90,16 @@ class Evaluation:
                 logger.error(f"Directory {self.dir_path} does not exist.")
                 raise NotADirectoryError
 
+        # Check version of Evaluation
+        if create_dir is False:
+            self.check_evaluation_version()
+
         # Create a local Spark Session if one is not provided.
         if not self.spark:
             logger.info("Creating a new Spark session.")
+            memory_info = psutil.virtual_memory()
+            driver_memory = 0.75 * memory_info.available / (1024**3)  # Use 75% of available memory
+            driver_maxresultsize = 0.5 * driver_memory
             conf = (
                 SparkConf()
                 .setAppName("TEEHR")
@@ -94,10 +110,18 @@ class Evaluation:
                 .set("spark.sql.execution.arrow.pyspark.enabled", "true")
                 .set("spark.sql.session.timeZone", "UTC")
                 .set("spark.driver.host", "localhost")
+                .set("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.4.1")
                 .set("spark.sql.parquet.enableVectorizedReader", "false")
+                .set("spark.driver.memory", f"{int(driver_memory)}g")
+                .set("spark.driver.maxResultSize", f"{int(driver_maxresultsize)}g")
             )
             self.spark = SparkSession.builder.config(conf=conf).getOrCreate()
 
+    @property
+    def generate(self) -> GeneratedTimeseries:
+        """The generate component class for generating synthetic data."""
+        return GeneratedTimeseries(self)
+
     @property
     def fetch(self) -> Fetch:
         """The fetch component class for accessing external data."""
@@ -347,3 +371,104 @@ class Evaluation:
             self.joined_timeseries.to_sdf().createOrReplaceTempView("joined_timeseries")
 
         return self.spark.sql(query)
+
+    def check_evaluation_version(self):
+        """Check the version of the TEEHR Evaluation."""
+        if self.is_s3:
+            fs = s3fs.S3FileSystem(anon=True)
+            version_file = self.dir_path.path + "/" + "version"
+        else:
+            fs = LocalFileSystem()
+            version_file = Path(self.dir_path, "version")
+        if not fs.exists(version_file):
+            logger.error(f"Version file not found in {self.dir_path}.")
+            if self.is_s3:
+                err_msg = (
+                    f"Please create a version file in {self.dir_path}."
+                )
+                logger.error(err_msg)
+                raise Exception(err_msg)
+            else:
+                # TODO: Change this to raise an error in v0.6.
+                version = teehr.__version__
+                with fs.open(version_file, "w") as f:
+                    f.write(version)
+                logger.info(
+                    f"Created version file in {self.dir_path}."
+                    " In the future this will raise an error."
+                )
+        else:
+            with fs.open(version_file) as f:
+                version_txt = str(f.read().strip())
+            match = re.findall(r'(\d+\.\d+\.\d+)', version_txt)  # Assumes semantic versioning
+            if len(match) != 1:
+                err_msg = f"Invalid version format in {self.dir_path}: {version_txt}"
+                logger.error(err_msg)
+                raise ValueError(err_msg)
+            else:
+                version = match[0]
+        # TODO: Uncomment this in v0.6
+        # if version < "0.6.0":
+        #     err_msg = (
+        #         f"Evaluation version {version} in {self.dir_path} is less than 0.6."
+        #         " Please run the migration to upgrade to the latest version."
+        #     )
+        #     logger.error(err_msg)
+        #     raise ValueError(err_msg)
+        # else:
+        #     # Update the version to the latest
+        #     pass
+        logger.info(
+            f"Found evaluation version {version} in {self.dir_path}."
+            " Future versions v0.6 and greater will require a conversion"
+            " to a new format."
+        )
+        return version
+
+    def filter(
+        self,
+        table_name: TableNamesEnum = None,
+        filters: Union[
+            str, dict, FilterBaseModel,
+            List[Union[str, dict, FilterBaseModel]]
+        ] = None,
+        table_filter: TableFilter = None
+    ) -> ps.DataFrame:
+        """Apply filters to a table returning a sdf.
+
+        Parameters
+        ----------
+        table_name: TableNamesEnum
+            The name of the table to filter. Defaults to None.
+        filters: Union[str, dict, FilterBaseModel, List[Union[str, dict, FilterBaseModel]]]
+            The filters to apply to the table. Defaults to None.
+        table_filter: TableFilter
+            A TableFilter object containing the table name and filters.
+            Defaults to None.
+        """
+        table_mapper = {
+            "primary_timeseries": self.primary_timeseries,
+            "secondary_timeseries": self.secondary_timeseries,
+            "locations": self.locations,
+            "units": self.units,
+            "variables": self.variables,
+            "configurations": self.configurations,
+            "attributes": self.attributes,
+            "location_attributes": self.location_attributes,
+            "location_crosswalks": self.location_crosswalks,
+            "joined_timeseries": self.joined_timeseries,
+        }
+        if table_filter is not None:
+            table_name = table_filter.table_name
+            filters = table_filter.filters
+        if table_name is None:
+            raise ValueError("Table name must be specified.")
+        base_table = table_mapper.get(table_name)
+        return validate_and_apply_filters(
+            sdf=base_table.to_sdf(),
+            filters=filters,
+            filter_model=base_table.filter_model,
+            fields_enum=base_table.field_enum(),
+            dataframe_schema=base_table._get_schema("pandas"),
+            validate=base_table.validate_filter_field_types
+        )