polytope-python 1.1.1__tar.gz → 2.0.1__tar.gz

polytope_python-2.0.1/.github/ci-config.yml

@@ -0,0 +1,13 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@refs/tags/v1.1.3
+  ecmwf/eccodes
+  ecmwf/eckit@develop
+  ecmwf/metkit
+  ecmwf/fdb
+  ecmwf/gribjump
+dependency_cmake_options: |
+  ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON"
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false

polytope_python-2.0.1/.github/ci-hpc-config.yml

@@ -0,0 +1,21 @@
+build:
+  python: "3.10"
+  modules:
+    - ninja
+    - aec
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/metkit@develop
+    - ecmwf/fdb@develop
+    - ecmwf/gribjump@develop
+  dependency_cmake_options:
+    - "ecmwf/gribjump: '-DENABLE_FDB_BUILD_TOOLS=ON'"
+    - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
+  parallel: 64
+  env:
+    - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+    - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+  pytest_cmd: |
+    python -m pytest -m 'not fdb' tests --cov=./ --cov-report=xml

polytope_python-2.0.1/.github/workflows/cd.yml

@@ -0,0 +1,30 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  pypi_binwheels:
+    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi-binwheel.yml@v2
+    secrets: inherit
+    with:
+      platforms: "['ubuntu-latest','macos-latest']"
+      pyversions: "['38','39','310','311','312']"
+      env_vars: |
+        {
+          "USE_RUST": "1",
+          "SETUPTOOLS_RUST_CARGO_PROFILE": "release"
+        }
+
+  pypi_purepython:
+    needs: pypi_binwheels
+    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit
+    with:
+      env_vars: |
+        {
+          "USE_RUST": "0",
+          "SETUPTOOLS_RUST_CARGO_PROFILE": "release"
+        }

polytope_python-2.0.1/.github/workflows/downstream-ci.yml

@@ -0,0 +1,40 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - "main"
+      - "develop"
+    tags-ignore:
+      - "**"
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }}
+      # codecov_upload: true
+      python_qa: true
+    secrets: inherit
+
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

polytope_python-2.0.1/.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

polytope_python-2.0.1/.github/workflows/test-pypi.yml

@@ -0,0 +1,32 @@
+
+name: test-cd
+
+on:
+  pull_request:
+    branches: [ "main"]
+
+jobs:
+  pypi_binwheels:
+    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi-binwheel.yml@v2
+    secrets: inherit
+    with:
+      platforms: "['ubuntu-latest','macos-latest']"
+      pyversions: "['38','39','310','311','312']"
+      testpypi: true
+      env_vars: |
+        {
+          "USE_RUST": "1",
+          "SETUPTOOLS_RUST_CARGO_PROFILE": "release"
+        }
+        
+  pypi_purepython:
+    needs: pypi_binwheels
+    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit
+    with:
+      testpypi: true
+      env_vars: |
+        {
+          "USE_RUST": "0",
+          "SETUPTOOLS_RUST_CARGO_PROFILE": "release"
+        }

polytope_python-2.0.1/.gitignore

@@ -0,0 +1,78 @@
+*.py[cod]
+*.swp
+*.grib
+*.gif
+*.html
+*.req
+*.prof
+*.idx
+*.grib
+*.xml
+*.pstats
+*.profile
+*.egg-info
+*.json
+*.txt
+*.gz
+
+.coverage
+.mypy_cache
+.vscode
+.pytest_cache
+
+__pycache__/
+.ipynb_checkpoints/
+
+_version.py
+target/
+dist/
+build/
+
+Cargo.lock
+.DS_Store
+
+site
+example_eo
+example_mri
+new_test_venv_polytope
+new_venv_polytope
+new_polytope_venv
+polytope_venv
+polytope_venv_latest
+new_updated_numpy_venv
+newest-polytope-venv
+serializedTree
+new_polytope_venv
+newest-polytope-venv
+new_polytope_venv
+polytope_feature/datacube/quadtree/target
+venv_python3_11
+venv_gj_iterator
+*.json
+venv_python3_11
+*.txt
+tests/data
+venv_gj_iterator
+target
+rust_deployment_venv
+*.so
+*.nc
+
+test_w_qubed
+rust_venv
+new_rust_test_venv
+*.so
+rust_deployment_venv
+*.lock
+**/target
+**/build
+_version.py
+*.DS_Store
+icon_grids
+rust_venv
+test_w_qubed
+performance_plots_qubed
+*.nc
+**/*venv*
+
+icon_grids

polytope_python-2.0.1/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+
+  - repo: https://github.com/psf/black
+    rev: 24.8.0 # Use the latest version of black
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1 # Use the latest version of flake8
+    hooks:
+      - id: flake8

polytope_python-2.0.1/.readthedocs.yaml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for MkDocs projects
+
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+
+# Required
+
+version: 2
+
+
+# Set the version of Python and other tools you might need
+
+build:
+
+  os: ubuntu-22.04
+
+  tools:
+
+    python: "3.12"
+
+
+mkdocs:
+
+  configuration: mkdocs.yml
+
+
+# Optionally declare the Python requirements required to build your docs
+
+python:
+
+  install:
+
+  - requirements: docs/requirements.txt

polytope_python-2.0.1/ACKNOWLEDGEMENTS.rst

@@ -0,0 +1,8 @@
+Acknowledgements
+================
+
+Destination Earth
+    This software is developed with co-funding by the European Union under the Destination Earth initiative.
+
+LEXIS 
+    This software is part of a project that has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 825532.

polytope_python-2.0.1/CONTRIBUTING.rst

@@ -0,0 +1,20 @@
+Contributing to *polytope*
+========================================
+
+Please report bug_ reports or pull-requests_ on GitHub_
+
+.. _bug: https://github.com/ecmwf/polytope/issues
+
+.. _pull-requests: https://github.com/ecmwf/polytope/pulls
+
+.. _GitHub: https://github.com/ecmwf/polytope
+
+We want your feedback, please e-mail: user-services@ecmwf.int
+
+The package is installed from PyPI with::
+
+    $ pip install polytope-python
+
+How you could use polytope:
+ * plot outputs, examples are provided in the examples folder
+ * go further than our examples and let us know how it goes

polytope_python-2.0.1/Makefile

@@ -0,0 +1,20 @@
+testk:
+	python3 -m pytest -vsrA tests/* -k $(filter-out $@, $(MAKECMDGOALS)) -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG
+
+testx:
+	python3 -m pytest -vsx tests/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG
+
+test:
+	python3 -m pytest -vsrA tests/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG
+
+examples:
+	python3 -m pytest -vsrA examples/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG
+
+performance:
+	python3 -m pytest -vsrA performance/* -W ignore::DeprecationWarning -W ignore::FutureWarning --log-cli-level=DEBUG
+
+docs:
+	mkdocs build
+	mkdocs serve
+
+.PHONY: init test docs

polytope_python-2.0.1/PKG-INFO

@@ -0,0 +1,248 @@
+Metadata-Version: 2.4
+Name: polytope-python
+Version: 2.0.1
+Summary: Polytope datacube feature extraction library
+Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
+Maintainer-email: James Hawkes <James.Hawkes@ecmwf.int>, Mathilde Leuridan <Mathilde.Leuridan@ecmwf.int>
+License: Apache License Version 2.0
+Project-URL: repository, https://github.com/ecmwf/polytope
+Project-URL: documentation, https://polytope.readthedocs.io/en/latest/
+Project-URL: issues, https://github.com/ecmwf/polytope/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: English
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas
+Requires-Dist: scipy
+Requires-Dist: sortedcontainers
+Requires-Dist: tripy
+Requires-Dist: xarray
+Requires-Dist: conflator
+Requires-Dist: protobuf
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-cov; extra == "tests"
+Requires-Dist: cffi; extra == "tests"
+Requires-Dist: eccodes; extra == "tests"
+Requires-Dist: h5netcdf; extra == "tests"
+Requires-Dist: h5py; extra == "tests"
+Requires-Dist: earthkit-data; extra == "tests"
+Requires-Dist: matplotlib; extra == "tests"
+Requires-Dist: pyfdb; extra == "tests"
+Dynamic: license-file
+
+
+<h3 align="center">
+<img src="./docs/images/polytope_logo_new_animated_AdobeExpress_3.gif" width=60%>
+</br>
+
+</h3>
+
+<p align="center">
+<a href="https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity">
+    <img src="https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity/incubating_badge.svg" alt="Project Maturity">
+  </a>
+<a href="https://github.com/ecmwf/codex/raw/refs/heads/main/ESEE"> <img src="https://github.com/ecmwf/codex/raw/refs/heads/main/ESEE/data_provision_badge.svg" alt="ESEE">
+</p>
+
+<p align="center">
+  <a href="https://github.com/ecmwf/polytope/actions/workflows/downstream-ci.yml">
+  <img src="https://github.com/ecmwf/polytope/actions/workflows/downstream-ci.yml/badge.svg" alt="ci">
+</a>
+  <a href="https://codecov.io/gh/ecmwf/polytope"><img src="https://codecov.io/gh/ecmwf/polytope/branch/develop/graph/badge.svg"></a>
+  <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg"></a>
+  <a href="https://github.com/ecmwf/polytope/releases"><img src="https://img.shields.io/github/v/release/ecmwf/polytope?color=blue&label=Release&style=flat-square"></a>
+  <a href='https://polytope.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/polytope/badge/?version=latest' alt='Documentation Status' /></a>
+</p>
+<p align="center">
+  <a href="#concept">Concept</a> •
+  <a href="#installation">Installation</a> •
+  <a href="#example">Example</a> •
+  <a href="#testing">Testing</a> •
+  <a href="https://polytope.readthedocs.io/en/latest/">Documentation</a>
+</p>
+
+Polytope is a library for extracting complex data from datacubes. It provides an API for non-orthogonal access to data, where the stencil used to extract data from the datacube can be any arbitrary *n*-dimensional polygon (called a *polytope*). This can be used to efficiently extract complex features from a datacube, such as polygon regions or spatio-temporal paths.
+
+Polytope is designed to extend different datacube backends:
+* XArray dataarrays
+* FDB object stores (through the GribJump software)
+
+Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **Polytope** can be used to dramatically decrease overall I/O load.
+
+> \[!IMPORTANT\]
+> This software is **Incubating** and subject to ECMWF's guidelines on [Software Maturity](https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity).
+
+<!-- > [!WARNING]
+> This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. -->
+
+## Concept 
+
+Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (*n*-dimensional polygon) stencil.
+
+<p align="center">
+  <img src="./docs/Algorithm/Overview/images_overview/ecmwf_polytope.png" alt="Polytope Concept" width="450"/>
+</p>
+
+The Polytope algorithm can for example be used to extract:
+
+- 2D cut-outs, such as country cut-outs, from a datacube
+    <p align="center">
+        <img src="./docs/images/greece.png" alt="Greece cut-out" width="250"/>
+    </p>
+
+- timeseries from a datacube
+    <p align="center">
+        <img src="./docs/images/timeseries.png" alt="Timeseries" width="350"/>
+    </p>
+
+- more complicated spatio-temporal paths, such as flight paths, from a datacube
+    <p align="center">
+        <img src="./docs/images/flight_path.png" alt="Flight path" width="350"/>
+    </p>
+
+- and many more high-dimensional shapes in arbitrary dimensions...
+
+For more information about the Polytope algorithm, refer to our [paper](https://arxiv.org/abs/2306.11553). 
+If this project is useful for your work, please consider citing this paper.
+
+## Installation 
+
+Install the polytope software with Python 3 (>=3.7) from GitHub directly with the command
+
+    python3 -m pip install git+ssh://git@github.com/ecmwf/polytope.git@develop
+
+or from PyPI with the command
+
+    python3 -m pip install polytope-python
+
+## Example
+
+Here is a step-by-step example of how to use this software.
+
+1. In this example, we first specify the data which will be in our Xarray datacube. Note that the data here comes from the GRIB file called "winds.grib", which is 3-dimensional with dimensions: step, latitude and longitude.
+    ```Python
+        import xarray as xr
+
+        array = xr.open_dataset("winds.grib", engine="cfgrib")
+    ```
+   
+    We then construct the Polytope object, passing in some additional metadata describing properties of the longitude axis.
+    ```Python
+        options = {"longitude": {"cyclic": [0, 360.0]}}
+
+        from polytope_feature.polytope import Polytope
+
+        p = Polytope(datacube=array, axis_options=options)
+    ```
+
+2. Next, we create a request shape to extract from the datacube.  
+  In this example, we want to extract a simple 2D box in latitude and longitude at step 0. We thus create the two relevant shapes we need to build this 3-dimensional object,
+    ```Python
+        import numpy as np
+        from polytope_feature.shapes import Box, Select
+
+        box = Box(["latitude", "longitude"], [0, 0], [1, 1])
+        step_point = Select("step", [np.timedelta64(0, "s")])
+    ```
+
+    which we then incorporate into a Polytope request.
+    ```Python
+        from polytope_feature.polytope import Request
+
+        request = Request(box, step_point)
+    ```
+
+3. Finally, extract the request from the datacube. 
+    ```Python
+        result = p.retrieve(request)
+    ```
+
+    The result is stored as an IndexTree containing the retrieved data organised hierarchically with axis indices for each point.
+    ```Python
+        result.pprint()
+        
+
+        Output IndexTree: 
+
+            ↳root=None
+                ↳step=0 days 00:00:00
+                        ↳latitude=0.0
+                                ↳longitude=0.0
+                                ↳longitude=1.0
+                        ↳latitude=1.0
+                                ↳longitude=0.0
+                                ↳longitude=1.0
+    ```
+
+## Testing
+
+#### Additional Dependencies
+
+The Polytope tests and examples require additional Python packages compared to the main Polytope algorithm.
+The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the tests and examples folders.
+Moreover, Polytope's tests and examples also require the installation of eccodes and GDAL.
+It is possible to install both of these dependencies using either a package manager or manually.
+
+## Contributing 
+
+The main repository is hosted on GitHub; testing, bug reports and contributions are highly welcomed and appreciated. 
+Please see the [Contributing](./CONTRIBUTING.rst) document for the best way to help. 
+
+Main contributors: 
+
+- Mathilde Leuridan - [ECMWF](https://www.ecmwf.int)
+- James Hawkes - [ECMWF](https://www.ecmwf.int)
+- Simon Smart - [ECMWF](www.ecmwf.int)
+- Emanuele Danovaro - [ECMWF](www.ecmwf.int)
+- Tiago Quintino - [ECMWF](www.ecmwf.int)
+
+See also the [contributors](https://github.com/ecmwf/polytope/contributors) for a more complete list. 
+
+## License 
+
+```
+Copyright 2021 European Centre for Medium-Range Weather Forecasts (ECMWF)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0).
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+In applying this licence, ECMWF does not waive the privileges and immunities
+granted to it by virtue of its status as an intergovernmental organisation nor
+does it submit to any jurisdiction.
+```
+
+## Citing 
+
+If this software is useful in your work, please consider citing our paper as 
+
+> Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., [“Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”](https://arxiv.org/abs/2306.11553), <i>arXiv e-prints</i>, 2023. doi:10.48550/arXiv.2306.11553.
+
+Other papers include: 
+
+> Leuridan, M., Bradley, C., Hawkes, J., Quintino, T., & Schultz, M. (2025, June). [Performance Analysis of an Efficient Algorithm for Feature Extraction from Large Scale Meteorological Data Stores.](https://dl.acm.org/doi/abs/10.1145/3732775.3733573) In Proceedings of the Platform for Advanced Scientific Computing Conference (pp. 1-9).
+
+## Acknowledgements
+
+Past and current funding and support for **Polytope** is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst).
+
+