geometallurgy 0.4.4__tar.gz → 0.4.12__tar.gz

{geometallurgy-0.4.4 → geometallurgy-0.4.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geometallurgy
-Version: 0.4.4
+Version: 0.4.12
 Summary: Tools for the geometallurgist
 Home-page: https://github.com/elphick/geometallurgy
 Author: Greg

{geometallurgy-0.4.4 → geometallurgy-0.4.12}/elphick/geomet/__init__.py

@@ -3,7 +3,6 @@ from importlib import metadata
 
 from .sample import Sample
 from .interval_sample import IntervalSample
-from .block_model import BlockModel
 
 try:
     __version__ = metadata.version('geometallurgy')

{geometallurgy-0.4.4 → geometallurgy-0.4.12}/elphick/geomet/base.py

@@ -2,23 +2,23 @@ import copy
 import inspect
 import logging
 import re
+import uuid
 from abc import ABC
 from pathlib import Path
-from typing import Optional, Union, Literal, TypeVar, TYPE_CHECKING, Any
+from typing import Optional, Union, Literal, TypeVar, TYPE_CHECKING
 
 import numpy as np
 import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
 
 from elphick.geomet.config import read_yaml
 from elphick.geomet.utils.components import get_components, is_compositional
 from elphick.geomet.utils.moisture import solve_mass_moisture
 from elphick.geomet.utils.pandas import mass_to_composition, composition_to_mass, composition_factors
-from elphick.geomet.utils.sampling import random_int
 from elphick.geomet.utils.timer import log_timer
 from .config.config_read import get_column_config
 from .plot import parallel_plot, comparison_plot
-import plotly.express as px
-import plotly.graph_objects as go
 
 if TYPE_CHECKING:
     from elphick.geomet.flowsheet.stream import Stream
@@ -122,7 +122,8 @@ class MassComposition(ABC):
                                                  mass_wet=self.mass_wet_var, mass_dry=self.mass_dry_var,
                                                  moisture_column_name=self.moisture_column,
                                                  component_columns=composition.columns,
-                                                 composition_units=self.composition_units)
+                                                 composition_units=self.composition_units,
+                                                 return_moisture=False)
             self._logger.debug(f"Data has been set.")
 
         else:
@@ -138,6 +139,22 @@ class MassComposition(ABC):
         # Recalculate the aggregate whenever the data changes
         self.aggregate = self.weight_average()
 
+    def get_mass_data(self, include_moisture: bool = True) -> pd.DataFrame:
+        """Get the mass data
+
+        Args:
+            include_moisture: If True (and moisture is in scope), include the moisture mass column
+
+        Returns:
+
+        """
+        if include_moisture and self.moisture_in_scope:
+            moisture_mass = self._mass_data[self.mass_wet_var] - self._mass_data[self.mass_dry_var]
+            mass_data: pd.DataFrame = self._mass_data.copy()
+            mass_data.insert(loc=2, column=self.moisture_column, value=moisture_mass)
+            return mass_data
+        return self._mass_data
+
     @property
     def aggregate(self) -> pd.DataFrame:
         if self._aggregate is None and self._mass_data is not None:
@@ -213,6 +230,172 @@ class MassComposition(ABC):
                 (self.mass_columns + [self.moisture_column] + self.composition_columns + self.supplementary_columns) if
                 col is not None]
 
+    def balance_composition(self, epsilon: float = 1.0e-6) -> MC:
+        """Balance the composition data
+
+        For records where the component mass exceeds the dry mass, the component masses are reduced proportionally
+        to equal the dry mass.  Records where the component mass is less than the dry mass are left unchanged.
+
+        epsilon: A small float to avoid component sums marginally over 100.0
+
+        Returns:
+            The object with balanced composition.
+        """
+        if self._mass_data is not None:
+            # calculate the ratio of the sum of the components to the dry mass
+            ratio = self._mass_data[self.composition_columns].sum(axis=1) / self._mass_data[self.mass_dry_var]
+            if ratio.max() <= 1.0:
+                return self
+
+            before_reduction = self._mass_data[self.composition_columns].copy()
+            # add a small value to the ratio to avoid component sums marginally over 100.0
+            ratio[ratio > 1.0] = ratio[ratio > 1.0] + epsilon
+            # to avoid reducing compliant records, clip the ratio at the lower side to 1.0
+            ratio = ratio.clip(lower=1.0)
+            # apply the ratio to the components
+            self._mass_data[self.composition_columns] = self._mass_data[self.composition_columns].div(ratio, axis=0)
+
+            # manage the reporting
+            affected_indexes = set(
+                self._mass_data.index[np.any(before_reduction != self._mass_data[self.composition_columns], axis=1)])
+            # log the action, including the first 50 indexes affected
+            affected_indexes_list = sorted(affected_indexes)[:50]
+
+            self.aggregate = self.weight_average()
+            self.status = OutOfRangeStatus(self, self.status.ranges)
+
+            self._logger.info(f"Object {self.name} has been balanced. "
+                              f"{len(affected_indexes)} records where composition has been reduced "
+                              f"to conform to dry mass. "
+                              f"Affected indexes (first 50): {affected_indexes_list}")
+        return self
+
+    def clip_recovery(self, other: MC, recovery_bounds: tuple[float, float] = (0.01, 0.99),
+                      allow_moisture_coercion: bool = True) -> MC:
+        """Clip the recovery to the specified bounds and recalculate the estimate.
+
+        Args:
+            other: The other MassComposition object, from which the recovery of self is calculated.
+            recovery_bounds: The bounds for the recovery between 0.0 and 1.0
+            allow_moisture_coercion: if True, allow the wet mass to be modified to maintain the moisture (in the
+            case that dry mass is clipped to manage recovery)
+
+        Returns:
+            The MassComposition object with the recovery clipped to the bounds.
+        """
+        recovery: pd.DataFrame = (self.get_mass_data(include_moisture=False) /
+                                  other.get_mass_data(include_moisture=False))
+
+        # Limit the recovery to the bounds
+        before_clip = recovery.copy()
+        recovery = recovery.clip(lower=recovery_bounds[0], upper=recovery_bounds[1]).fillna(0.0)
+
+        # Check if any records were affected
+        affected_indexes = set(recovery.index[np.any(before_clip != recovery, axis=1)])
+        if affected_indexes:
+            # Recalculate the estimate from the bound recovery
+            new_mass: pd.DataFrame = recovery * other.get_mass_data(include_moisture=False)[recovery.columns]
+
+            if self.moisture_in_scope and allow_moisture_coercion:
+                # Calculate the moisture from the new mass
+                new_mass[self.mass_wet_var] = solve_mass_moisture(mass_dry=new_mass[self.mass_dry_var],
+                                                                  moisture=self.data[self.moisture_column])
+
+            # Log the top 50 records affected by the recovery coercion
+            affected_indexes_list = sorted(affected_indexes)[:50]
+            self._logger.info(f"Recovery coercion affected {len(affected_indexes)} records. "
+                              f"Affected indexes (first 50): {affected_indexes_list}")
+
+            # Update the mass data of self
+            self.update_mass_data(new_mass)
+        else:
+            self._logger.info("Recovery coercion did not affect any records.")
+
+        return self
+
+    def set_moisture(self, moisture: Union[pd.Series, float, int], mass_to_adjust: Literal['wet', 'dry'] = 'wet') -> MC:
+        """Set the moisture to the specified value
+
+        A convenience method for an mc object that modifies the concrete mass to deliver the specified moisture.
+
+        Args:
+            moisture: The moisture value to set.  Can be a constant or series.
+            mass_to_adjust: The mass to adjust, either 'wet' or 'dry'.
+
+        Returns:
+
+        """
+
+        if not self.moisture_in_scope:
+            raise AssertionError("This method is not applicable unless moisture_in_scope property is True.")
+
+        if isinstance(moisture, float) or isinstance(moisture, int):
+            # create a series with the same index as the mass data
+            moisture = pd.Series(float(moisture), index=self._mass_data.index)
+        elif not isinstance(moisture, pd.Series):
+            raise TypeError(f"moisture must be a float or a pd.Series, not {type(moisture)}")
+
+        if mass_to_adjust == 'wet':
+            self._mass_data[self.mass_wet_var] = solve_mass_moisture(mass_dry=self._mass_data[self.mass_dry_var],
+                                                                     moisture=moisture)
+        elif mass_to_adjust == 'dry':
+            self._mass_data[self.mass_dry_var] = solve_mass_moisture(mass_wet=self._mass_data[self.mass_wet_var],
+                                                                     moisture=moisture)
+        else:
+            raise ValueError(f"mass_to_adjust must be 'wet' or 'dry', not {mass_to_adjust}")
+
+        self.aggregate = self.weight_average()
+        self.status = OutOfRangeStatus(self, self.status.ranges)
+
+        return self
+
+    def clip_composition(self, ranges: Optional[dict[str, list[float]]] = None, epsilon: float = 1.0e-05) -> MC:
+        """Clip the components
+
+        Clip to the components to within the range provided or the default range for each component.
+        This method does not clip moisture - see set_moisture and solve_moisture for that.
+
+        Args:
+            ranges: An optional dict defining a list of [lo, hi] floats for each component.  If not provided,
+            the default range from the config file will be used.
+            epsilon: A small float to ensure the clipped values lie marginally inside the specified range.
+
+        Returns:
+            The object with clipped composition.
+        """
+
+        # load the default ranges from the config file
+        component_ranges: dict = self._get_component_ranges(ranges)
+
+        # define a small value to ensure the clipped values lie marginally inside the specified range.
+        # clip the components
+        affected_indexes = set()
+        for component, component_range in component_ranges.items():
+            before_clip = self._mass_data[component].copy()
+            # add a small value to the range to ensure the clipped values lie marginally inside the specified range.
+            component_range = [component_range[0] + epsilon, component_range[1] - epsilon]
+            # define the component mass that aligns with the lower and upper bounds
+            component_mass_limits = self._mass_data[self.mass_dry_var].values[:, np.newaxis] * np.array(
+                component_range) / self.composition_factor
+            # apply the clip to the mass data
+            self._mass_data[component] = self._mass_data[component].clip(lower=component_mass_limits[:, 0],
+                                                                         upper=component_mass_limits[:, 1])
+            affected_indexes.update(self._mass_data.index[before_clip != self._mass_data[component]])
+
+        # log the action, including the first 50 indexes affected
+        affected_indexes_list = sorted(affected_indexes)[:50]
+
+        self.aggregate = self.weight_average()
+        self.status = OutOfRangeStatus(self, self.status.ranges)
+
+        if len(affected_indexes) > 0:
+            self._logger.info(f"Object {self.name} has been clipped. "
+                              f"{len(affected_indexes)} records where composition has been clipped to the "
+                              f"range: {component_ranges}. "
+                              f"Affected indexes (first 50): {affected_indexes_list}")
+
+        return self
+
     def plot_parallel(self, color: Optional[str] = None,
                       vars_include: Optional[list[str]] = None,
                       vars_exclude: Optional[list[str]] = None,
@@ -352,7 +535,8 @@ class MassComposition(ABC):
 
             # Create a DataFrame from the weighted averages
             weighted_averages_df = pd.concat([mass_sum, composition], axis=1)
-        else:
+
+        else:  # group by a variable
             group_var: pd.Series = self._supplementary_data[group_by]
             weighted_averages_df = self._mass_data.groupby(group_var).apply(
                 lambda x: pd.DataFrame(
@@ -447,7 +631,7 @@ class MassComposition(ABC):
                 non_mass_cols: list[str] = [col for col in value.columns if
                                             col not in [self.mass_wet_var, self.mass_dry_var, self.moisture_var, 'h2o',
                                                         'H2O', 'H2O']]
-                component_cols: list[str] = get_components(value[non_mass_cols], strict=False)
+                component_cols: list[str] = get_components(value[non_mass_cols].columns, strict=False)
             else:
                 component_cols: list[str] = self.component_vars
             composition = value[component_cols]
@@ -479,6 +663,7 @@ class MassComposition(ABC):
                 self._supplementary_data.index = self._mass_data.index
             self._supplementary_data = self._supplementary_data.loc[value.index]
         self.aggregate = self.weight_average()
+        self.status = OutOfRangeStatus(self, self.status.ranges)
 
     def filter_by_index(self, index: pd.Index):
         """Update the data by index"""
@@ -529,8 +714,8 @@ class MassComposition(ABC):
         comp: 'Stream'
 
         # create the relationships
-        ref.nodes = [self.nodes[1], random_int()]
-        comp.nodes = [self.nodes[1], random_int()]
+        ref.nodes = [self.nodes[1], uuid.uuid4()]
+        comp.nodes = [self.nodes[1], uuid.uuid4()]
 
         return ref, comp
 
@@ -549,6 +734,10 @@ class MassComposition(ABC):
 
         res: MC = self.create_congruent_object(name=name, include_mc_data=True,
                                                include_supp_data=include_supplementary_data)
+
+        if set(self._mass_data.columns) != set(other._mass_data.columns):
+            raise ValueError(f"Mass data columns do not match: {set(self._mass_data.columns)} != "
+                             f"{set(other._mass_data.columns)}")
         res.update_mass_data(self._mass_data + other._mass_data)
 
         # Ensure self and other are Stream objects
@@ -558,7 +747,7 @@ class MassComposition(ABC):
 
         # create the relationships
         other.nodes = [other.nodes[0], self.nodes[1]]
-        res.nodes = [self.nodes[1], random_int()]
+        res.nodes = [self.nodes[1], uuid.uuid4()]
 
         return res
 
@@ -576,6 +765,11 @@ class MassComposition(ABC):
         """
         res = self.create_congruent_object(name=name, include_mc_data=True,
                                            include_supp_data=include_supplementary_data)
+
+        if set(self._mass_data.columns) != set(other._mass_data.columns):
+            raise ValueError(f"Mass data columns do not match: {set(self._mass_data.columns)} != "
+                             f"{set(other._mass_data.columns)}")
+
         res.update_mass_data(self._mass_data - other._mass_data)
 
         # Ensure self and other are Stream objects
@@ -585,7 +779,8 @@ class MassComposition(ABC):
         other: 'Stream'
 
         # create the relationships
-        res.nodes = [self.nodes[1], random_int()]
+        other.nodes = [self.nodes[1], other.nodes[1]]
+        res.nodes = [self.nodes[1], uuid.uuid4()]
 
         return res
 
@@ -696,7 +891,7 @@ class MassComposition(ABC):
                             moisture_column_name: Optional[str] = None,
                             component_columns: Optional[list[str]] = None,
                             composition_units: Literal['%', 'ppm', 'ppb'] = '%',
-                            **kwargs):
+                            **kwargs) -> MC:
         """
         Class method to create a MassComposition object from a mass dataframe.
 
@@ -737,6 +932,41 @@ class MassComposition(ABC):
 
         return res
 
+    def compare(self, other: 'MassComposition', comparisons: Union[str, list[str]] = 'recovery',
+                explicit_names: bool = True) -> pd.DataFrame:
+
+        comparisons = [comparisons] if isinstance(comparisons, str) else comparisons
+        valid_comparisons: set = {'recovery', 'difference', 'divide', 'all'}
+
+        cols = [col for col in self.data.data_vars if col not in self.data.mc.mc_vars_attrs]
+
+        chunks: list[pd.DataFrame] = []
+        if 'recovery' in comparisons or comparisons == ['all']:
+            df: pd.DataFrame = self._mass_data[self.component_vars] / other._mass_data[self.component_vars]
+            if explicit_names:
+                df.columns = [f"{self.name}_{col}_{self.config['comparisons']['recovery']}_{other.name}" for col in
+                              df.columns]
+            chunks.append(df)
+        if 'difference' in comparisons or comparisons == ['all']:
+            df: pd.DataFrame = self.data[cols] - other.data[cols]
+            if explicit_names:
+                df.columns = [f"{self.name}_{col}_{self.config['comparisons']['difference']}_{other.name}" for col in
+                              df.columns]
+            chunks.append(df)
+        if 'divide' in comparisons or comparisons == ['all']:
+            df: pd.DataFrame = self.data[cols] / other.data[cols]
+            if explicit_names:
+                df.columns = [f"{self.name}_{col}_{self.config['comparisons']['divide']}_{other.name}" for col in
+                              df.columns]
+            chunks.append(df)
+
+        if not chunks:
+            raise ValueError(f"The comparison argument is not valid: {valid_comparisons}")
+
+        res: pd.DataFrame = pd.concat(chunks, axis=1)
+
+        return res
+
     def reset_index(self, index_name: str) -> MC:
         res = self.create_congruent_object(name=f"{self.name} (reset_index)", include_mc_data=True,
                                            include_supp_data=True)
@@ -749,6 +979,20 @@ class MassComposition(ABC):
 
         return res
 
+    def _get_component_ranges(self, ranges: dict[str, list]) -> dict[str, list]:
+
+        d_ranges: dict = get_column_config(config_dict=self.config, var_map=self.variable_map,
+                                           config_key='range')
+        # filter to include only components
+        d_ranges = {k: v for k, v in d_ranges.items() if k in self.composition_columns}
+
+        # modify the default dict based on any user passed constraints
+        if ranges:
+            for k, v in ranges.items():
+                d_ranges[k] = v
+
+        return d_ranges
+
 
 class OutOfRangeStatus:
     """A class to check and report out-of-range records in an MC object."""
@@ -767,7 +1011,7 @@ class OutOfRangeStatus:
             self.oor: pd.DataFrame = self._check_range()
             self.num_oor: int = len(self.oor)
             self.failing_components: Optional[list[str]] = list(
-                self.oor.dropna(axis=1).columns) if self.num_oor > 0 else None
+                self.oor.dropna(axis=1, how='all').columns) if self.num_oor > 0 else None
 
     def get_ranges(self, ranges: dict[str, list]) -> dict[str, list]:
 
@@ -811,3 +1055,79 @@ class OutOfRangeStatus:
         if isinstance(other, OutOfRangeStatus):
             return self.oor.equals(other.oor)
         return False
+
+
+class SampleStatus:
+    """A class to check and report sample status in an MC object.
+
+    A MassComposition object (Sample, Stream, etc) can be unhealthy if:
+    1. the total mass of components is greater than the dry mass
+    2. any of the masses are negative
+    3. the component values are out of range
+    """
+
+    def __init__(self, mc: 'MC', component_limits: dict):
+        """Initialize with an MC object."""
+        self._logger = logging.getLogger(__name__)
+        self.mc: 'MC' = mc
+        self.sample_status: Optional[dict] = None
+        self.num_samples: Optional[int] = None
+        self.failing_samples: Optional[list[str]] = None
+
+        if mc.mass_data is not None:
+            self.sample_status = self.get_sample_status(component_limits)
+            self.num_samples = len(self.sample_status)
+            self.failing_samples = [sample for sample, status in self.sample_status.items() if not status['ok']]
+
+    def get_sample_status(self, component_limits: dict) -> dict:
+        """Check if all records are within the constraints."""
+        sample_status = {}
+        if self.mc._mass_data is not None:
+            df: pd.DataFrame = self.mc.data[self.mc.composition_columns]
+            mass_dry = self.mc._mass_data[self.mc.mass_dry_var]
+
+            # Check for component limits
+            for component, limits in component_limits.items():
+                oor = df[(df[component] < limits[0]) | (df[component] > limits[1])]
+                for sample in oor.index:
+                    if sample not in sample_status:
+                        sample_status[sample] = {'ok': True, 'causes': []}
+                    sample_status[sample]['ok'] = False
+                    sample_status[sample]['causes'].append(f"{component} out of range")
+
+            # Check if total mass of components is greater than dry mass
+            total_component_mass = df.sum(axis=1)
+            for sample in total_component_mass.index:
+                if sample not in sample_status:
+                    sample_status[sample] = {'ok': True, 'causes': []}
+                if total_component_mass[sample] > mass_dry[sample]:
+                    sample_status[sample]['ok'] = False
+                    sample_status[sample]['causes'].append("Total mass of components greater than dry mass")
+
+            # Check for negative masses
+            negative_masses = df[df < 0].dropna(how='all')
+            for sample in negative_masses.index:
+                if sample not in sample_status:
+                    sample_status[sample] = {'ok': True, 'causes': []}
+                sample_status[sample]['ok'] = False
+                sample_status[sample]['causes'].append("Negative mass detected")
+        return sample_status
+
+    @property
+    def ok(self) -> bool:
+        """Return True if all records are within range, False otherwise."""
+        if self.num_samples > 0:
+            self._logger.warning(f'{self.num_samples} unhealthy samples exist.')
+        return True if self.num_samples == 0 else False
+
+    def __str__(self) -> str:
+        """Return a string representation of the status."""
+        res: str = f'status.ok: {self.ok}\n'
+        res += f'num_samples: {self.num_samples}'
+        return res
+
+    def __eq__(self, other: object) -> bool:
+        """Return True if other Status has the same out-of-range records."""
+        if isinstance(other, SampleStatus):
+            return self.sample_status == other.sample_status
+        return False

geometallurgy-0.4.12/elphick/geomet/config/flowsheet_example_partition.yaml

@@ -0,0 +1,31 @@
+FLOWSHEET:
+  flowsheet:
+    name: Flowsheet
+  streams:  # graph edges
+    Feed:
+      name: Feed
+      node_in: feed
+      node_out: screen
+    Coarse:
+      name: Coarse
+      node_in: screen
+      node_out: lump
+    Fine:
+      name: Fine
+      node_in: screen
+      node_out: fines
+  operations:  # graph nodes
+    feed:
+      name: feed
+    screen:
+      name: screen
+      type: PartitionOperation
+      partition:
+        module: elphick.geomet.utils.partition
+        function: napier_munn_size_1mm
+        partition_stream: Lump  # the stream that the partition model defines
+        args: null  # e.g. d50, ep if not defined in the (partial) function
+    lump:
+      name: lump
+    fines:
+      name: fines

{geometallurgy-0.4.4 → geometallurgy-0.4.12}/elphick/geomet/config/mc_config.yml

@@ -27,6 +27,7 @@ MC:
       - [from, to]
       - [retained, passing]
       - [sink, float]
+      - [lo, hi]
   comparisons:
     recovery: 'rec'
     difference: 'diff'