chaid-segmenter 0.1.0__tar.gz

chaid_segmenter-0.1.0/CHAID/__init__.py

@@ -0,0 +1,8 @@
+from .split import Split
+from .tree import Tree
+from .node import Node
+from .column import NominalColumn, OrdinalColumn, ContinuousColumn
+from .stats import Stats
+from .invalid_split_reason import InvalidSplitReason
+
+__version__ = "5.4.3"

chaid_segmenter-0.1.0/CHAID/__main__.py

@@ -0,0 +1,110 @@
+"""
+This package provides a python implementation of the Chi-Squared Automatic
+Inference Detection (CHAID) decision tree.
+"""
+import argparse
+from .tree import Tree
+import pandas as pd
+import numpy as np
+
+
+def main():
+    """Entry point when module is run from command line"""
+
+    parser = argparse.ArgumentParser(description='Run the chaid algorithm on a'
+                                     ' csv/sav file.')
+    parser.add_argument('file')
+    parser.add_argument('dependent_variable', nargs=1)
+    parser.add_argument('--dependent-variable-type', type=str)
+
+    var = parser.add_argument_group('Independent Variable Specification')
+    var.add_argument('nominal_variables', nargs='*', help='The names of '
+                     'independent variables to use that have no intrinsic '
+                     'order to them')
+    var.add_argument('--ordinal-variables', type=str, nargs='*',
+                     help='The names of independent variables to use that '
+                     'have an intrinsic order but a finite amount of states')
+    parser.add_argument('--weights', type=str, help='Name of weight column')
+
+    parser.add_argument('--max-depth', type=int, help='Max depth of generated '
+                        'tree')
+    parser.add_argument('--min-parent-node-size', type=int, help='Minimum number of '
+                        'samples required to split the parent node')
+    parser.add_argument('--min-child-node-size', type=int, help='Minimum number of '
+                        'samples required to split the child node')
+    parser.add_argument('--alpha-merge', type=float, help='Alpha Merge')
+    group = parser.add_mutually_exclusive_group(required=False)
+    group.add_argument('--classify', action='store_true', help='Add column to'
+                       ' input with the node id of the node that that '
+                       'respondent has been placed into')
+    group.add_argument('--predict', action='store_true', help='Add column to '
+                       'input with the value of the  dependent variable that '
+                       'the majority of respondents in that node selected')
+    group.add_argument('--rules', action='store_true')
+    group.add_argument('--export', action='store_true', help='Whether to export the chart to pdf/dot')
+    group.add_argument('--export-path', type=str, help='Path to store chart output')
+
+    group.add_argument('--exhaustive', action='store_true', help='To implement exhustive CHAID')
+
+    nspace = parser.parse_args()
+
+    if nspace.file[-4:] == '.csv':
+        data = pd.read_csv(nspace.file)
+    elif nspace.file[-4:] == '.sav':
+        import savReaderWriter as spss
+        raw_data = spss.SavReader(nspace.file, returnHeader=True)
+        raw_data_list = list(raw_data)
+        data = pd.DataFrame(raw_data_list)
+        data = data.rename(columns=data.loc[0]).iloc[1:]
+    else:
+        print('Unknown file type')
+        exit(1)
+
+    config = {}
+    if nspace.max_depth:
+        config['max_depth'] = nspace.max_depth
+    if nspace.alpha_merge:
+        config['alpha_merge'] = nspace.alpha_merge
+    if nspace.min_parent_node_size:
+        config['min_parent_node_size'] = nspace.min_parent_node_size
+    if nspace.min_child_node_size:
+        config['min_child_node_size'] = nspace.min_child_node_size
+    if nspace.weights:
+        config['weight'] = nspace.weights
+    if nspace.dependent_variable_type:
+        config['dep_variable_type'] = nspace.dependent_variable_type
+    if nspace.exhaustive:
+        config['is_exhaustive'] = nspace.exhaustive
+
+    ordinal = nspace.ordinal_variables or []
+    nominal = nspace.nominal_variables or []
+    independent_variables = nominal + ordinal
+    types = dict(zip(nominal + ordinal, ['nominal'] * len(nominal) + ['ordinal'] * len(ordinal)))
+    if len(independent_variables) == 0:
+        print('Need to provide at least one independent variable')
+        exit(1)
+    tree = Tree.from_pandas_df(data, types, nspace.dependent_variable[0],
+                               **config)
+
+    if nspace.export or nspace.export_path:
+        tree.render(nspace.export_path, True)
+
+    if nspace.classify:
+        predictions = pd.Series(tree.node_predictions())
+        predictions.name = 'node_id'
+        data = pd.concat([data, predictions], axis=1)
+        print(data.to_csv())
+    elif nspace.predict:
+        predictions = pd.Series(tree.model_predictions())
+        predictions.name = 'predicted'
+        data = pd.concat([data, predictions], axis=1)
+        print(data.to_csv())
+    elif nspace.rules:
+        print('\n'.join(str(x) for x in tree.classification_rules()))
+    else:
+        tree.print_tree()
+        print('Accuracy: ', tree.accuracy())
+
+
+if __name__ == "__main__":
+    main()

chaid_segmenter-0.1.0/CHAID/column.py

@@ -0,0 +1,333 @@
+import numpy as np
+from math import isnan
+from itertools import combinations
+from .mapping_dict import MappingDict
+
+
+def convert_to_python_type(value):
+    """
+    Convert numpy scalar types to Python native types.
+    This ensures compatibility with numpy >= 2.0.0 where numpy scalars
+    are preserved in operations like np.unique().
+
+    Parameters
+    ----------
+    value : any
+        The value to convert (may be a numpy scalar or Python native type)
+
+    Returns
+    -------
+    The value converted to a Python native type if it was a numpy scalar,
+    otherwise the original value
+    """
+    if hasattr(value, 'item'):
+        # numpy scalars have an item() method that returns the Python scalar
+        return value.item()
+    return value
+
+def is_sorted(ndarr, nan_val=None):
+    store = []
+    for arr in ndarr:
+        if arr == [] or len(arr) == 1: continue
+        if nan_val is not None and nan_val in arr:
+            arr.remove(nan_val)
+        store.append(arr[-1] - arr[0] == len(arr) - 1)
+    return all(store)
+
+class Column(object):
+    """
+    A numpy array with metadata
+
+    Parameters
+    ----------
+    arr : iterable object
+        The numpy array
+    metadata : dict
+        The substitutions of the vector
+    missing_id : string
+        An identifier for the missing value to be associated
+    substitute : bool
+        Whether the objects in the given array need to be substitued for
+        integers
+    """
+    def __init__(self, arr=None, metadata=None, missing_id='<missing>',
+                 substitute=True, weights=None, name=None):
+        self.metadata = dict(metadata or {})
+        self.arr = np.array(arr)
+        self._missing_id = missing_id
+        self.weights = weights
+        self.name = name
+
+    def __iter__(self):
+        return iter(self.arr)
+
+    def __getitem__(self, key):
+        raise NotImplementedError
+
+    def __setitem__(self, key, value):
+        raise NotImplementedError
+
+    def possible_groupings(self):
+        raise NotImplementedError
+
+    @property
+    def type(self):
+        """
+        Returns a string representing the type
+        """
+        raise NotImplementedError
+
+    def deep_copy(self):
+        """
+        Returns a deep copy
+        """
+        raise NotImplementedError
+
+    def bell_set(self, collection, ordinal=False):
+        """
+        Calculates the Bell set
+        """
+        if len(collection) == 1:
+            yield [ collection ]
+            return
+
+        first = collection[0]
+        for smaller in self.bell_set(collection[1:]):
+            for n, subset in enumerate(smaller):
+                if not ordinal or (ordinal and is_sorted(smaller[:n] + [[ first ] + subset] + smaller[n+1:], self._nan)):
+                    yield smaller[:n] + [[ first ] + subset] + smaller[n+1:]
+
+            if not ordinal or (ordinal and is_sorted([ [ first ] ] + smaller, self._nan)):
+                yield [ [ first ] ] + smaller
+
+
+class NominalColumn(Column):
+    """
+    A column containing numerical values that are unrelated to
+    one another (i.e. do not follow a progression)
+    """
+    def __init__(self, arr=None, metadata=None, missing_id='<missing>',
+                 substitute=True, weights=None, name=None):
+        super(self.__class__, self).__init__(arr, metadata=metadata, missing_id=missing_id, weights=weights, name=name)
+        if substitute and metadata is None:
+            self.substitute_values(arr)
+
+        self._groupings = MappingDict()
+        for x in np.unique(self.arr):
+            self._groupings[x] = [x]
+
+    def deep_copy(self):
+        """
+        Returns a deep copy.
+        """
+        return NominalColumn(self.arr, metadata=self.metadata, name=self.name,
+                             missing_id=self._missing_id, substitute=False, weights=self.weights)
+
+    def substitute_values(self, vect):
+        """
+        Internal method to substitute integers into the vector, and construct
+        metadata to convert back to the original vector.
+
+        np.nan is always given -1, all other objects are given integers in
+        order of apperence.
+
+        Parameters
+        ----------
+        vect : np.array
+            the vector in which to substitute values in
+        """
+
+        try:
+            unique = np.unique(vect)
+        except:
+            unique = set(vect)
+
+        unique = [
+            x for x in unique if not isinstance(x, float) or not isnan(x)
+        ]
+
+        arr = np.copy(vect)
+        for new_id, value in enumerate(unique):
+            np.place(arr, arr==value, new_id)
+            # Convert value to Python native type for numpy 2.0 compatibility
+            self.metadata[new_id] = convert_to_python_type(value)
+        arr = arr.astype(np.float64)
+        np.place(arr, np.isnan(arr), -1)
+        self.arr = arr
+
+        if -1 in arr:
+            self.metadata[-1] = self._missing_id
+
+    def __getitem__(self, key):
+        new_weights = None if self.weights is None else self.weights[key]
+        return NominalColumn(self.arr[key], metadata=self.metadata, substitute=False, weights=new_weights, name=self.name)
+
+    def __setitem__(self, key, value):
+        self.arr[key] = value
+        return self
+
+    def groups(self):
+        # Convert all values in groups to Python native types for numpy 2.0 compatibility
+        return [[convert_to_python_type(item) for item in group] for group in self._groupings.values()]
+
+    def possible_groupings(self):
+        return combinations(self._groupings.keys(), 2)
+
+    def all_combinations(self):
+        bell_set = self.bell_set(sorted(list(self._groupings.keys())))
+        next(bell_set)
+        return bell_set
+
+    def group(self, x, y):
+        self._groupings[x] += self._groupings[y]
+        del self._groupings[y]
+        self.arr[self.arr == y] = x
+
+    @property
+    def type(self):
+        """
+        Returns a string representing the type
+        """
+        return 'nominal'
+
+
+class OrdinalColumn(Column):
+    """
+    A column containing integer values that have an order
+    """
+    def __init__(self, arr=None, metadata=None, missing_id='<missing>',
+                 groupings=None, substitute=True, weights=None, name=None):
+        super(self.__class__, self).__init__(arr, metadata, missing_id=missing_id, weights=weights, name=name)
+        self._nan = np.iinfo(np.int64).min
+
+        if substitute and metadata is None:
+            self.arr, self.orig_type = self.substitute_values(self.arr)
+        elif substitute and metadata and not np.issubdtype(self.arr.dtype, np.integer):
+            # custom metadata has been passed in from external source, and must be converted to int
+            self.arr = self.arr.astype(int)
+            self.metadata = { int(k):v for k, v in metadata.items() }
+            self.metadata[self._nan] = missing_id
+
+        self._groupings = {}
+        if groupings is None:
+            for x in np.unique(self.arr):
+                self._groupings[x] = [x, x + 1, False]
+        else:
+            for x in np.unique(self.arr):
+                self._groupings[x] = list(groupings[x])
+        self._possible_groups = None
+
+    def substitute_values(self, vect):
+        if not np.issubdtype(vect.dtype, np.integer):
+            uniq = set(vect)
+            uniq_floats = np.array(list(uniq), dtype=float)
+            uniq_ints = uniq_floats.astype(int)
+            nan = self._missing_id
+            self.metadata = {
+                new: nan if isnan(as_float) else old
+                for old, as_float, new in zip(uniq, uniq_floats, uniq_ints)
+            }
+            self.arr = self.arr.astype(float)
+        return self.arr.astype(int), self.arr.dtype.type
+
+    def deep_copy(self):
+        """
+        Returns a deep copy.
+        """
+        return OrdinalColumn(self.arr, metadata=self.metadata, name=self.name,
+                             missing_id=self._missing_id, substitute=True,
+                             groupings=self._groupings, weights=self.weights)
+
+    def __getitem__(self, key):
+        new_weights = None if self.weights is None else self.weights[key]
+        return OrdinalColumn(self.arr[key], metadata=self.metadata, name=self.name,
+                             missing_id=self._missing_id, substitute=True,
+                             groupings=self._groupings, weights=new_weights)
+
+    def __setitem__(self, key, value):
+        self.arr[key] = value
+        return self
+
+    def groups(self):
+        vals = self._groupings.values()
+        return [
+            [convert_to_python_type(x) for x in range(minmax[0], minmax[1])] + ([convert_to_python_type(self._nan)] if minmax[2] else [])
+            for minmax in vals
+        ]
+
+    def possible_groupings(self):
+        if self._possible_groups is None:
+            ranges = sorted(self._groupings.items())
+            candidates = zip(ranges[0:], ranges[1:])
+            self._possible_groups = [
+                (k1, k2) for (k1, minmax1), (k2, minmax2) in candidates
+                if minmax1[1] == minmax2[0]
+            ]
+            if self._nan in self.arr:
+                self._possible_groups += [
+                    (key, self._nan) for key in self._groupings.keys() if key != self._nan
+                ]
+        return self._possible_groups.__iter__()
+
+    def all_combinations(self):
+        bell_set = self.bell_set(sorted(list(self._groupings.keys())), True)
+        next(bell_set)
+        return bell_set
+
+
+    def group(self, x, y):
+        self._possible_groups = None
+        if y != self._nan:
+            x = int(x)
+            y = int(y)
+            x_max = self._groupings[x][1]
+            y_min = self._groupings[y][0]
+            if y_min >= x_max:
+                self._groupings[x][1] = self._groupings[y][1]
+            else:
+                self._groupings[x][0] = y_min
+            self._groupings[x][2] = self._groupings[x][2] or self._groupings[y][2]
+        else:
+            self._groupings[x][2] = True
+
+        del self._groupings[y]
+        self.arr[self.arr == y] = x
+
+    @property
+    def type(self):
+        """
+        Returns a string representing the type
+        """
+        return 'ordinal'
+
+class ContinuousColumn(Column):
+    """
+    A column containing numerical values on a continuous scale
+    """
+    def __init__(self, arr=None, metadata=None, missing_id='<missing>',
+                 weights=None):
+        if not np.issubdtype(arr.dtype, np.number):
+            raise ValueError('Must only pass numerical values to create continuous column')
+
+        super(self.__class__, self).__init__(np.nan_to_num(arr), metadata, missing_id=missing_id, weights=weights)
+
+    def deep_copy(self):
+        """
+        Returns a deep copy.
+        """
+        return ContinuousColumn(self.arr, metadata=self.metadata, missing_id=self._missing_id, weights=self.weights)
+
+    def __getitem__(self, key):
+        new_weights = None if self.weights is None else self.weights[key]
+        return ContinuousColumn(self.arr[key], metadata=self.metadata, weights=new_weights)
+
+    def __setitem__(self, key, value):
+        self.arr[key] = value
+        return self
+
+    @property
+    def type(self):
+        """
+        Returns a string representing the type
+        """
+        return 'continuous'

chaid_segmenter-0.1.0/CHAID/graph.py

@@ -0,0 +1,124 @@
+import os
+import warnings
+from datetime import datetime
+
+# Wrap all the optional imports
+try:
+    import plotly.graph_objs as go
+    import plotly.io as pio
+    import colorlover as cl
+    from graphviz import Digraph
+    
+except ImportError:
+    warnings.warn(UserWarning('Imports of optional packages needed to generate graphs failed. Please install with the "graph" option.'))
+    go = None
+    pio = None
+    cl = None
+    Digraph = None
+
+    FIG_BASE = {}
+    FIG_BASE_DATA = {}
+    TABLE_HEADER = []
+    TABLE_CONFIG = {}
+    TABLE_CELLS_CONFIG = {}
+
+else:
+    FIG_BASE = {
+        "layout": {
+            "margin_t": 50,
+            "annotations": [{"font_size": 18, "x": 0.5, "y": 0.5}],
+        },
+    }
+    FIG_BASE_DATA = {
+        "domain": {"x": [0, 1], "y": [0.4, 1.0]},
+        "hole": 0.4,
+        "type": "pie",
+        "marker_colors": cl.scales["5"]["qual"]["Set1"],
+    }
+    TABLE_HEADER = ["<i>p</i>", "score", "splitting on"]
+    TABLE_CONFIG = {
+        "domain": {"x": [0.3, 0.7], "y": [0, 0.37]},
+        "header": {"fill_color": "#FFF"},
+    }
+    TABLE_CELLS_CONFIG = {
+        "line_color": "#FFF",
+        "align": "left",
+        "font_color": "#282828",
+        "height": 27,
+        "fill_color": ["#EBC1EE", "#EDEAFB"],
+    }    
+
+try:
+    # Python 3.2 and newer
+    from tempfile import TemporaryDirectory
+except ImportError:
+    # minimal backport of TemporaryDirectory for Python 2.7, sufficient
+    # for use with this module.
+    import shutil
+    from tempfile import mkdtemp
+    class TemporaryDirectory(object):
+        def __init__(self):
+            self.name = mkdtemp()
+        def __enter__(self):
+            return self.name
+        def __exit__(self, *args):
+            shutil.rmtree(self.name, ignore_errors=True)
+
+
+class Graph(object):
+    """
+    Visualisation of the tree
+
+    Parameters
+    ----------
+    tree : iterable CHAID tree
+    """
+
+    def __init__(self, tree):
+        self.tree = tree
+
+    def render(self, path, view):
+        if path is None:
+            path = os.path.join("trees", "{:%Y-%m-%d %H:%M:%S}.gv".format(datetime.now()))
+        with TemporaryDirectory() as self.tempdir:
+            g = Digraph(
+                format="png",
+                graph_attr={"splines": "ortho"},
+                node_attr={"shape": "plaintext", "labelloc": "b"},
+            )
+            for node in self.tree:
+                image = self.bar_chart(node)
+                g.node(str(node.node_id), image=image)
+                if node.parent is not None:
+                    edge_label = "     ({})     \n ".format(', '.join(map(str, node.choices)))
+                    g.edge(str(node.parent), str(node.node_id), xlabel=edge_label)
+            g.render(path, view=view)
+
+    def bar_chart(self, node):
+        fig = dict(
+            data=[
+                dict(
+                    values=list(node.members.values()),
+                    labels=list(node.members),
+                    showlegend=(node.node_id == 0),
+                    **FIG_BASE_DATA
+                )
+            ],
+            **FIG_BASE
+        )
+
+        if not node.is_terminal:
+            fig["data"].append(self._table(node))
+
+        filename = os.path.join(self.tempdir, "node-{}.png".format(node.node_id))
+        pio.write_image(fig, file=filename, format="png")
+        return filename
+
+    def _table(self, node):
+        p = None if node.p is None else format(node.p, ".5f")
+        score = None if node.score is None else format(node.score, ".2f")
+        values = [p, score, node.split.column]
+        return go.Table(
+            cells=dict(values=[TABLE_HEADER, values], **TABLE_CELLS_CONFIG),
+            **TABLE_CONFIG
+        )

chaid_segmenter-0.1.0/CHAID/invalid_split_reason.py

@@ -0,0 +1,18 @@
+from enum import Enum, unique
+
+@unique
+class InvalidSplitReason(Enum):
+    """
+    Private class to store possible invalid reasons
+    """
+    ALPHA_MERGE =          'p-value greater than alpha merge'
+    MIN_CHILD_NODE_SIZE =  'splitting would create nodes with less than the minimum child ' \
+                           'node size'
+    MAX_DEPTH =            'the max depth has been reached'
+    MAX_SPLITS =           'number of splits greater than maximum specified'
+    MIN_PARENT_NODE_SIZE = 'the minimum parent node size threshold has been reached'
+    PURE_NODE =            'the node only contains single category respondents'
+    NODE_NOT_EXHAUSTIVE =  'the node is not exhaustive'
+
+    def __str__(self):
+        return self.value

chaid_segmenter-0.1.0/CHAID/mapping_dict.py

@@ -0,0 +1,5 @@
+class MappingDict(dict):
+    """ a dict with a default value when no key present """
+    def __missing__(self, key):
+        value = self[key] = [key]
+        return value