eo-tides 0.6.3.dev6__tar.gz → 0.6.3.dev8__tar.gz

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/.gitignore

@@ -1,6 +1,7 @@
 # Ignore project-specific directories
 tests/data/tide_models
 tests/data/tide_models_clipped
+tests/data/tide_models_synthetic
 docs/source
 
 # From https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eo-tides
-Version: 0.6.3.dev6
+Version: 0.6.3.dev8
 Summary: Tide modelling tools for large-scale satellite earth observation analysis
 Project-URL: Homepage, https://GeoscienceAustralia.github.io/eo-tides/
 Project-URL: Repository, https://github.com/GeoscienceAustralia/eo-tides

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/docs/changelog.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v0.6.3
+
+### New features
+
+- Major updates to installation and setup documentation, to provide additional context about tide modelling data and installing `eo-tides` using both `pip` and `uv`
+
+### Bug fixes
+
+- Fixed bug where ensemble tide modelling used excessive memory, by ensuring dtype of ensemble modelled tides matches dtype of input modelled tides ([#70](https://github.com/GeoscienceAustralia/eo-tides/pull/70))
+- Added missing `dask` dependency to requirements ([#68](https://github.com/GeoscienceAustralia/eo-tides/pull/68))
+
 ## v0.6.2
 
 ### New features

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/eo_tides/model.py

@@ -330,6 +330,9 @@ def ensemble_tides(
     x = tide_df.index.get_level_values(level="x")
     y = tide_df.index.get_level_values(level="y")
 
+    # Identify input datatype
+    input_dtype = tide_df.tide_height.dtype
+
     # Load model ranks points and reproject to same CRS as x and y
     model_ranking_cols = [f"rank_{m}" for m in ensemble_models]
     try:
@@ -373,6 +376,7 @@ def ensemble_tides(
         .set_index(["tide_model", "x", "y"])
         .groupby(["x", "y"])
         .rank()
+        .astype("float32")  # use smaller dtype for rankings to save memory
     )
 
     # If no custom ensemble funcs are provided, use a default ensemble
@@ -404,9 +408,12 @@ def ensemble_tides(
 
         # Use weightings to combine multiple models into single ensemble
         ensemble_df = (
-            # Calculate weighted mean and convert back to dataframe
+            # Calculate weighted mean
             grouped.weighted.sum()
             .div(grouped.weights.sum())
+            # Make sure datatype is the same as the input
+            .astype(input_dtype)
+            # Convert to dataframe
             .to_frame("tide_height")
             # Label ensemble model and ensure indexes are in expected order
             .assign(tide_model=ensemble_n)

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eo-tides"
-version = "0.6.3.dev6"
+version = "0.6.3.dev8"
 description = "Tide modelling tools for large-scale satellite earth observation analysis"
 authors = [
     { name = "Robbi Bishop-Taylor" },

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/tests/test_model.py

@@ -3,7 +3,7 @@ import pandas as pd
 import pytest
 from pyTMD.compute import tide_elevations
 
-from eo_tides.model import _parallel_splits, _set_directory, model_phases, model_tides
+from eo_tides.model import _parallel_splits, _set_directory, ensemble_tides, model_phases, model_tides
 from eo_tides.validation import eval_metrics
 
 GAUGE_X = 122.2183
@@ -432,6 +432,30 @@ def test_model_tides_ensemble():
     ])
 
 
+# Test ensemble dtype is set correctly
+@pytest.mark.parametrize(
+    "dtype",
+    ["float32", "float64", "int16"],
+)
+def test_model_tides_ensemble_dtype(dtype):
+    # Create dummy modelled tide data with specific dtype
+    modelled_tides_df = pd.DataFrame({
+        "time": pd.date_range(start="2000-01-01", periods=5, freq="5h").repeat(2),
+        "x": 122.2183,
+        "y": -18.0008,
+        "tide_model": ["EOT20", "HAMTIDE11"] * 5,
+        "tide_height": np.random.uniform(-4, 3, 10).astype(dtype),
+    })
+    modelled_tides_df = modelled_tides_df.set_index(["time", "x", "y"])
+
+    # Run ensemble modelling on modelled tides input
+    ensemble_df = ensemble_tides(modelled_tides_df, ensemble_models=ENSEMBLE_MODELS, crs="EPSG:4326")
+
+    # Verify that output tides match are as expected, and match the iput data
+    assert ensemble_df.tide_height.dtype == dtype
+    assert ensemble_df.tide_height.dtype == modelled_tides_df.tide_height.dtype
+
+
 @pytest.mark.parametrize("time_offset", ["15 min", "20 min"])
 def test_model_phases(time_offset):
     phase_df = model_phases(

{eo_tides-0.6.3.dev6 → eo_tides-0.6.3.dev8}/tests/testing.ipynb

@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -44,6 +44,15 @@
     "ENSEMBLE_MODELS = [\"EOT20\", \"HAMTIDE11\"]  # simplified for tests"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cd .."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -53,7 +62,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,7 +117,7 @@
     "\n",
     "    # Load measured tides from ABSLMP tide gauge data\n",
     "    measured_tides_df = pd.read_csv(\n",
-    "        \"../tests/data/IDO71013_2020.csv\",\n",
+    "        \"tests/data/IDO71013_2020.csv\",\n",
     "        index_col=0,\n",
     "        parse_dates=True,\n",
     "        na_values=-9999,\n",
@@ -220,23 +229,6 @@
     "create_synthetic_hamtide11()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/home/jovyan/Robbi/eo-tides\n"
-     ]
-    }
-   ],
-   "source": [
-    "cd .."
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -250,7 +242,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && pytest tests/test_model.py --verbose -k test_model_tides_ensemble"
+    "!export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && pytest tests/test_model.py --verbose -k test_model_tides_ensemble_dtype"
    ]
   },
   {
@@ -282,53 +274,81 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
-      "platform linux -- Python 3.10.15, pytest-8.3.3, pluggy-1.5.0 -- /env/bin/python3.10\n",
-      "cachedir: .pytest_cache\n",
-      "rootdir: /home/jovyan/Robbi/eo-tides\n",
-      "configfile: pyproject.toml\n",
-      "plugins: anyio-4.6.2.post1, nbval-0.11.0\n",
-      "collected 29 items / 20 deselected / 9 selected                                \u001b[0m\u001b[1m\n",
-      "\n",
-      "tests/test_utils.py::test_clip_models \u001b[32mPASSED\u001b[0m\u001b[33m                             [ 11%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[EOT20-bbox0-point0-hawaii] \u001b[32mPASSED\u001b[0m\u001b[33m [ 22%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[EOT20-bbox1-point1-uk] \u001b[32mPASSED\u001b[0m\u001b[33m [ 33%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[EOT20-bbox2-point2-aus] \u001b[32mPASSED\u001b[0m\u001b[33m [ 44%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[EOT20-bbox3-point3-pacific] \u001b[32mPASSED\u001b[0m\u001b[33m [ 55%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[HAMTIDE11-bbox4-point4-hawaii] \u001b[32mPASSED\u001b[0m\u001b[33m [ 66%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[HAMTIDE11-bbox5-point5-uk] \u001b[32mPASSED\u001b[0m\u001b[33m [ 77%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[HAMTIDE11-bbox6-point6-aus] \u001b[32mPASSED\u001b[0m\u001b[33m [ 88%]\u001b[0m\n",
-      "tests/test_utils.py::test_clip_models_bbox[HAMTIDE11-bbox7-point7-pacific] \u001b[32mPASSED\u001b[0m\u001b[33m [100%]\u001b[0m\n",
-      "\n",
-      "\u001b[33m=============================== warnings summary ===============================\u001b[0m\n",
-      "<frozen importlib._bootstrap>:241\n",
-      "  <frozen importlib._bootstrap>:241: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility. Expected 16 from C header, got 96 from PyObject\n",
-      "\n",
-      "tests/test_utils.py: 36 warnings\n",
-      "  /env/lib/python3.10/site-packages/pyproj/transformer.py:817: DeprecationWarning: Conversion of an array with ndim > 0 to a scalar is deprecated, and will error in future. Ensure you extract a single element from your array before performing this operation. (Deprecated NumPy 1.25.)\n",
-      "    return self._transformer._transform_point(\n",
-      "\n",
-      "tests/test_utils.py::test_clip_models_bbox[EOT20-bbox0-point0-hawaii]\n",
-      "tests/test_utils.py::test_clip_models_bbox[HAMTIDE11-bbox4-point4-hawaii]\n",
-      "  /home/jovyan/Robbi/eo-tides/eo_tides/model.py:125: UserWarning: On-the-fly cropping is not compatible with the provided clipped model files; running with `crop=False`.\n",
-      "    warnings.warn(\n",
-      "\n",
-      "-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html\n",
-      "\u001b[33m=========== \u001b[32m9 passed\u001b[0m, \u001b[33m\u001b[1m20 deselected\u001b[0m, \u001b[33m\u001b[1m39 warnings\u001b[0m\u001b[33m in 74.74s (0:01:14)\u001b[0m\u001b[33m ===========\u001b[0m\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "!export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && pytest tests/test_utils.py --verbose -k test_clip_models"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Ensemble dtype"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from eo_tides.model import model_tides\n",
+    "from eo_tides.model import ensemble_tides\n",
+    "\n",
+    "# Set modelling location based on bbox centroid\n",
+    "time = pd.date_range(start=\"2000-01\", end=\"2001-03\", freq=\"5h\")\n",
+    "\n",
+    "# Model using unclipped vs clipped files\n",
+    "tide_df = model_tides(\n",
+    "    x=GAUGE_X,\n",
+    "    y=GAUGE_Y,\n",
+    "    time=time,\n",
+    "    model=\"all\",\n",
+    "    directory=\"./tests/data/tide_models\",\n",
+    ")\n",
+    "\n",
+    "# tide_df[\"tide_height\"] = tide_df.tide_height.astype(\"float64\")\n",
+    "# ensemble_df = ensemble_tides(tide_df, ensemble_models=[\"EOT20\", \"HAMTIDE11\"], crs=\"EPSG:4326\")\n",
+    "\n",
+    "# print(ensemble_df)\n",
+    "# print(ensemble_df.dtypes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(tide_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "# Create data with a cross product approach\n",
+    "data = pd.DataFrame({\n",
+    "    'time': pd.date_range(start='2000-01-01', periods=5, freq='5h').repeat(2),\n",
+    "    'x': 122.2183,\n",
+    "    'y': -18.0008,\n",
+    "    'tide_model': ['EOT20', 'HAMTIDE11'] * 5,\n",
+    "    'tide_height': np.random.uniform(-4, 3, 10).astype(\"float32\")\n",
+    "})\n",
+    "\n",
+    "# Set multi-index\n",
+    "data = data.set_index(['time', 'x', 'y'])\n",
+    "\n",
+    "print(data)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -338,44 +358,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Preparing to clip suitable NetCDF models: ['HAMTIDE11']\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Clipping HAMTIDE11: 100%|██████████| 9/9 [00:00<00:00, 16.72it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " ✅ Clipped model exported and verified\n",
-      "\n",
-      "Outputs exported to tests/data/tide_models_synthetic_aus\n",
-      "────────────────────────────────────────────────────────────────────────────────\n",
-      " 󠀠🌊  | Model                | Expected path                                                           \n",
-      "────────────────────────────────────────────────────────────────────────────────\n",
-      " ✅  │ HAMTIDE11            │ tests/data/tide_models_synthetic_aus/hamtide                            \n",
-      "────────────────────────────────────────────────────────────────────────────────\n",
-      "\n",
-      "Summary:\n",
-      "Available models: 1/52\n",
-      "Modelling tides with HAMTIDE11\n",
-      "Modelling tides with HAMTIDE11\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "from eo_tides.utils import clip_models\n",
     "from eo_tides.model import model_tides\n",
@@ -2777,7 +2762,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -2791,7 +2776,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.12.0"
   }
  },
  "nbformat": 4,