omfpandas 0.1.1__tar.gz

omfpandas-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Greg Elphick
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

omfpandas-0.1.1/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.1
+Name: omfpandas
+Version: 0.1.1
+Summary: 
+Home-page: https://github.com/elphick/omfpandas
+Author: Greg
+Author-email: 11791585+elphick@users.noreply.github.com
+Requires-Python: >=3.9,<3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Provides-Extra: io
+Requires-Dist: omf (>=1.0.1,<2.0.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: pyarrow (>=16.1.0,<17.0.0) ; extra == "io"
+Project-URL: Documentation, https://elphick.github.io/omfpandas
+Project-URL: Repository, https://github.com/elphick/omfpandas
+Description-Content-Type: text/markdown
+
+# omfpandas
+
+[![Run Tests](https://github.com/Elphick/omfpandas/actions/workflows/poetry_build_and_test.yml/badge.svg?branch=main)](https://github.com/Elphick/omfpandas/actions/workflows/poetry_build_and_test.yml)
+[![Publish Docs](https://github.com/Elphick/omfpandas/actions/workflows/poetry_sphinx_docs_to_gh_pages.yml/badge.svg?branch=main)](https://github.com/Elphick/omfpandas/actions/workflows/poetry_sphinx_docs_to_gh_pages.yml)
+
+A pandas (and parquet) interface for the [Open Mining Format package (omf)](https://omf.readthedocs.io/en/latest/).
+
+When working with OMF files, it is often useful to convert the data to a pandas DataFrame.
+This package provides a simple interface to do so.
+
+The parquet format is a nice, compact, efficient format to persist pandas DataFrames.
+This package also provides a simple interface to convert an omf element to a parquet file.
+When datasets do not fit into memory, parquet files can be read in chunks or by column.
+
+## Installation
+
+```bash
+pip install omfpandas
+```
+
+If you intend to use the parquet functionality, you will need to install the optional dependencies.
+
+```bash
+pip install omfpandas[io]
+```
+
+## Roadmap
+
+- [ ] 0.2.0 - Add support for reading a VolumeElement (Block Model) from an OMF file as a pandas DataFrame. 
+  Export a VolumeElement as a parquet file.
+- [ ] 0.3.0 - Add support for writing a DataFrame to an OMF VolumeElement
+- [ ] 0.4.0 - Add support for low-memory/out-of-core writing an omf element to parquet
+- [ ] ...
+

omfpandas-0.1.1/README.md

@@ -0,0 +1,33 @@
+# omfpandas
+
+[![Run Tests](https://github.com/Elphick/omfpandas/actions/workflows/poetry_build_and_test.yml/badge.svg?branch=main)](https://github.com/Elphick/omfpandas/actions/workflows/poetry_build_and_test.yml)
+[![Publish Docs](https://github.com/Elphick/omfpandas/actions/workflows/poetry_sphinx_docs_to_gh_pages.yml/badge.svg?branch=main)](https://github.com/Elphick/omfpandas/actions/workflows/poetry_sphinx_docs_to_gh_pages.yml)
+
+A pandas (and parquet) interface for the [Open Mining Format package (omf)](https://omf.readthedocs.io/en/latest/).
+
+When working with OMF files, it is often useful to convert the data to a pandas DataFrame.
+This package provides a simple interface to do so.
+
+The parquet format is a nice, compact, efficient format to persist pandas DataFrames.
+This package also provides a simple interface to convert an omf element to a parquet file.
+When datasets do not fit into memory, parquet files can be read in chunks or by column.
+
+## Installation
+
+```bash
+pip install omfpandas
+```
+
+If you intend to use the parquet functionality, you will need to install the optional dependencies.
+
+```bash
+pip install omfpandas[io]
+```
+
+## Roadmap
+
+- [ ] 0.2.0 - Add support for reading a VolumeElement (Block Model) from an OMF file as a pandas DataFrame. 
+  Export a VolumeElement as a parquet file.
+- [ ] 0.3.0 - Add support for writing a DataFrame to an OMF VolumeElement
+- [ ] 0.4.0 - Add support for low-memory/out-of-core writing an omf element to parquet
+- [ ] ...

omfpandas-0.1.1/omfpandas/__init__.py

@@ -0,0 +1,10 @@
+from omfpandas.omfp import OMFPandas
+import omfpandas.volume
+
+from importlib import metadata
+
+try:
+    __version__ = metadata.version('omfpandas')
+except metadata.PackageNotFoundError:
+    # Package is not installed
+    pass

omfpandas-0.1.1/omfpandas/omfp.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+from omf import OMFReader
+
+from omfpandas.volume import volume_to_df, volume_to_parquet
+
+
+class OMFPandas:
+    def __init__(self, omf_filepath: Path):
+        """Instantiate the OMFPandas object.
+
+        :param omf_filepath: Path to the OMF file.
+        """
+        if not omf_filepath.exists():
+            raise FileNotFoundError(f'File not found: {omf_filepath}')
+        elif not omf_filepath.suffix == '.omf':
+            raise ValueError(f'File is not an OMF file: {omf_filepath}')
+        self.omf_filepath = omf_filepath
+        self._elements = OMFReader(str(omf_filepath)).get_project().elements
+        self.elements: dict[str, str] = {e.name: e.subtype for e in self._elements}
+
+    def get_element_by_name(self, element_name: str):
+        """Get an element by its name.
+
+        :param element_name: The name of the element to retrieve.
+        :return:
+        """
+        element = [e for e in self._elements if e.name == element_name]
+        if not element:
+            raise ValueError(f"Element '{element_name}' not found in the OMF file: {self.omf_filepath.name}. "
+                             f"Available elements are: {list(self.elements.keys())}")
+        elif len(element) > 1:
+            raise ValueError(f"Multiple elements with the name '{element_name}' found in the OMF file: "
+                             f"{self.omf_filepath.name}")
+        return element[0]
+
+    def volume_to_df(self, volume_name: str, variables: Optional[list[str]] = None,
+                     with_geometry_index: bool = True) -> pd.DataFrame:
+        """Return a DataFrame from a VolumeElement.
+
+        Only variables assigned to the `cell` (as distinct from the grid `points`) are loaded.
+
+        :param volume_name: The name of the VolumeElement to convert.
+        :param variables: The variables to include in the DataFrame.  If None, all variables are included.
+        :param with_geometry_index:
+        :return: The DataFrame representing the VolumeElement.
+        """
+        volume = self.get_element_by_name(volume_name)
+        # check the element retrieved is the expected type
+        if volume.__class__.__name__ != 'VolumeElement':
+            raise ValueError(f"Element '{volume}' is not a VolumeElement in the OMF file: {self.omf_filepath}")
+
+        return volume_to_df(volume, variables=variables, with_geometry_index=with_geometry_index)
+
+    def volume_to_parquet(self, volume_name: str, parquet_filepath: Path):
+        """Write a VolumeElement to a Parquet file.
+
+        :param volume_name: The name of the VolumeElement to convert.
+        :param parquet_filepath: The path to the Parquet file to write.
+        :return:
+        """
+        volume = self.get_element_by_name(volume_name)
+        # check the element retrieved is the expected type
+        if volume.__class__.__name__ != 'VolumeElement':
+            raise ValueError(f"Element '{volume}' is not a VolumeElement in the OMF file: {self.omf_filepath}")
+
+        volume_to_parquet(volume, parquet_filepath)

omfpandas-0.1.1/omfpandas/volume.py

@@ -0,0 +1,81 @@
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from omf import VolumeElement, OMFReader, VolumeGridGeometry
+
+
+def volume_to_df(volume: VolumeElement, variables: Optional[list[str]] = None,
+                 with_geometry_index: bool = True) -> pd.DataFrame:
+    """Convert volume to a DataFrame."""
+    # read the data
+    df: pd.DataFrame = read_volume_variables(volume, variables=variables)
+    if with_geometry_index:
+        df.index = get_index(volume)
+    return df
+
+
+def volume_to_parquet(volume: VolumeElement, out_path: Optional[Path] = None, variables: Optional[list[str]] = None,
+                      with_geometry_index: bool = True):
+    """Convert volume to a Parquet file."""
+    if out_path is None:
+        out_path = Path(f"{volume.name}.parquet")
+    df: pd.DataFrame = volume_to_df(volume, variables=variables, with_geometry_index=with_geometry_index)
+    df.to_parquet(out_path)
+
+
+def read_volume_variables(volume: VolumeElement, variables: list[str]) -> pd.DataFrame:
+    """Read the variables from the VolumeElement."""
+    # identify 'cell' variables in the file
+    variables = [v.name for v in volume.data if v.location == 'cells']
+
+    # Loop over the variables
+    chunks: list[np.ndarray] = []
+    for variable in variables:
+        # Check if the variable exists in the VolumeElement
+        if variable not in variables:
+            raise ValueError(f"Variable '{variable}' not found in the VolumeElement: {element}")
+        chunks.append(_get_variable_data_by_name(volume, variable).ravel())
+
+    # Concatenate all chunks into a single DataFrame
+    return pd.DataFrame(np.vstack(chunks), index=variables).T
+
+
+def get_index(volume: VolumeElement) -> pd.MultiIndex:
+    """Returns a pd.MultiIndex for the volume element."""
+    geometry: VolumeGridGeometry = volume.geometry
+    ox, oy, oz = geometry.origin
+
+    # Make coordinates (points) along each axis, i, j, k
+    i = ox + np.cumsum(geometry.tensor_u)
+    i = np.insert(i, 0, ox)
+    j = oy + np.cumsum(geometry.tensor_v)
+    j = np.insert(j, 0, oy)
+    k = oz + np.cumsum(geometry.tensor_w)
+    k = np.insert(k, 0, oz)
+
+    # convert to centroids
+    x, y, z = (i[1:] + i[:-1]) / 2, (j[1:] + j[:-1]) / 2, (k[1:] + k[:-1]) / 2
+    xx, yy, zz = np.meshgrid(x, y, z, indexing="ij")
+
+    # Calculate dx, dy, dz
+    dxx, dyy, dzz = np.meshgrid(geometry.tensor_u, geometry.tensor_v, geometry.tensor_w, indexing="ij")
+
+    # TODO: consider rotation
+
+    index = pd.MultiIndex.from_arrays([xx.ravel("F"), yy.ravel("F"), zz.ravel("F"),
+                                       dxx.ravel("F"), dyy.ravel("F"), dzz.ravel("F")],
+                                      names=['x', 'y', 'z', 'dx', 'dy', 'dz'])
+
+    return index
+
+
+def _get_variable_data_by_name(volume: VolumeElement, variable_name: str) -> np.ndarray:
+    # get the index of the variable in order to index into elements
+    scalar_data = [sd for sd in volume.data if sd.location == 'cells' and sd.name == variable_name]
+    if not scalar_data:
+        raise ValueError(f"Variable '{variable_name}' not found as cell data in the VolumeElement: {volume}")
+    elif len(scalar_data) > 1:
+        raise ValueError(f"Multiple variables with the name '{variable_name}' found in the VolumeElement: {volume}")
+    return scalar_data[0].array.array

omfpandas-0.1.1/pyproject.toml

@@ -0,0 +1,38 @@
+[tool.poetry]
+name = "omfpandas"
+packages = [{ include = "omfpandas" }]
+version = "0.1.1"
+description = ""
+authors = ["Greg <11791585+elphick@users.noreply.github.com>"]
+repository = "https://github.com/elphick/omfpandas"
+documentation = "https://elphick.github.io/omfpandas"
+readme = "README.md"
+
+[[tool.poetry.source]]
+name = "PyPI"
+priority = "primary"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.11"
+pandas = "^2.2.2"
+pyarrow = { version = "^16.1.0", optional = true }
+omf = "^1.0.1"
+
+[tool.poetry.extras]
+io = ["pyarrow"]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.2.1"
+sphinx = "^7.3.7"
+sphinx-gallery = "^0.16.0"
+sphinx-rtd-theme = "^2.0.0"
+pyarrow = "^16.1.0"
+ydata-profiling = "^4.8.3"
+coverage = "^7.5.3"
+towncrier = "^23.11.0"
+myst-parser = "^3.0.1"
+sphinx-autodoc-typehints = "^2.1.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"