simple-sklearn 0.1.0__py3-none-any.whl

simple_sklearn/__init__.py

@@ -0,0 +1,25 @@
+"""Simple implementations of standard machine learning algorithms.
+
+This library provides implementations of various standard machine learning models.
+Designed to integrate seamlessly with scikit-learn's
+[estimator API](https://scikit-learn.org/stable/developers/develop.html),
+all estimators in this package inherit from
+[`sklearn.base.BaseEstimator`][] and the appropriate mixins
+([`ClassifierMixin`][sklearn.base.ClassifierMixin] or [`ClusterMixin`][sklearn.base.ClusterMixin]).
+
+This package is divided into two primary subpackages:
+ - [`classification`][simple_sklearn.classification]:
+    Supervised machine learning algorithms for classification tasks.
+ - [`clustering`][simple_sklearn.clustering]:
+    Unsupervised machine learning algorithms for clustering tasks.
+"""
+
+from . import classification, clustering
+
+try:
+    # _version.py file is auto-generated by setuptools_scm during the build process
+    from ._version import __version__
+except ImportError:
+    __version__ = "unknown"
+
+__all__ = ["__version__", "classification", "clustering"]

simple_sklearn/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

simple_sklearn/classification/__init__.py

@@ -0,0 +1,28 @@
+"""Supervised machine learning algorithms for classification tasks.
+
+This subpackage contains implementations of standard classification models.
+All estimators in this module inherit from
+[`sklearn.base.BaseEstimator`][] and [`sklearn.base.ClassifierMixin`][].
+
+Available models:
+ - [`OneRClassifier`][simple_sklearn.classification.OneRClassifier]:
+    1R (One Rule) classification.
+ - [`NaiveBayesClassifier`][simple_sklearn.classification.NaiveBayesClassifier]:
+    Categorical Naive Bayes classification.
+ - [`KNeighborsClassifier`][simple_sklearn.classification.KNeighborsClassifier]:
+    K-Nearest Neighbors classification.
+ - [`DecisionTreeClassifier`][simple_sklearn.classification.DecisionTreeClassifier]:
+    Decision Tree classification using the ID3 algorithm.
+"""
+
+from ._decision_tree import DecisionTreeClassifier
+from ._k_neighbors import KNeighborsClassifier
+from ._naive_bayes import NaiveBayesClassifier
+from ._one_r import OneRClassifier
+
+__all__ = [
+    "DecisionTreeClassifier",
+    "KNeighborsClassifier",
+    "NaiveBayesClassifier",
+    "OneRClassifier",
+]

simple_sklearn/classification/_decision_tree.py

@@ -0,0 +1,210 @@
+"""Decision Tree Classification.
+
+This module provides the `DecisionTreeClassifier` class and its associated node structures.
+"""
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from numpy.typing import NDArray
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.utils.multiclass import type_of_target
+from sklearn.utils.validation import check_is_fitted, validate_data
+from typing_extensions import Self
+
+
+class DecisionTreeClassifier(ClassifierMixin, BaseEstimator):  # type: ignore
+    """Perform Decision Tree classification using the ID3 algorithm.
+
+    The Decision Tree classifier builds a tree from categorical features by recursively
+    partitioning the data based on the feature that maximizes information gain
+    (decreases entropy).
+
+    Note:
+        This implementation is strictly for discrete/categorical feature data.
+        Passing continuous features will result in severe overfitting.
+
+    Attributes:
+        classes_: The unique class labels observed in the training data.
+        num_features_: The number of features seen during fitting.
+        feature_unique_values_: A list of sets containing the unique values encountered for each feature.
+        tree_: The root node of the fitted decision tree (a `DecisionTreeNode` instance).
+            Nodes store integer-encoded labels mapped to the `classes_` array.
+    """
+
+    classes_: NDArray[Any]
+    num_features_: int
+    feature_unique_values_: list[set[Any]]
+    tree_: "DecisionTreeNode"
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def fit(self, X: Any, y: Any) -> Self:
+        """Fit the Decision Tree classification model.
+
+        Args:
+            X: Training instances to fit the model. Can be an array-like of shape `(n_samples, n_features)`.
+            y: Target values (class labels) for the training instances. Array-like of shape `(n_samples,)`.
+
+        Returns:
+            The fitted instance.
+
+        Raises:
+            ValueError: If `y` is of a continuous type.
+        """
+        X, y = validate_data(self, X, y)
+        X = np.array(X)
+
+        if type_of_target(y) in ("continuous", "continuous-multioutput"):
+            raise ValueError(f"Unknown label type: {type_of_target(y)}")
+        self.classes_, y = np.unique(y, return_inverse=True)
+
+        self.num_features_ = X.shape[1]
+        feat_indices = list(range(self.num_features_))
+        df = pd.DataFrame(X, columns=feat_indices)
+        df["y"] = y
+
+        self.feature_unique_values_ = [set(df[feat_index]) for feat_index in feat_indices]
+        self.tree_ = self._id3_algorithm(df, set(feat_indices))
+
+        return self
+
+    def predict(self, X: Any) -> NDArray[Any]:
+        """Predict class labels for the given input data.
+
+        Traverses the fitted decision tree for each sample. If an unseen feature value
+        is encountered at a split node, the model falls back to predicting the majority
+        class label of that specific node.
+
+        Args:
+            X: Instances to predict. Can be an array-like of shape `(n_samples, n_features)`.
+
+        Returns:
+            An array of shape `(n_samples,)` containing the predicted class labels for each sample.
+        """
+        check_is_fitted(self)
+        X = validate_data(self, X, reset=False)
+        X = np.array(X)
+
+        y_pred = []
+        for x in X:
+            curr_node = self.tree_
+            while True:
+                match curr_node:
+                    case LeafNode():
+                        y_pred.append(curr_node.label)
+                        break
+                    case SplitterNode():
+                        feat_value = x[curr_node.feat_index]
+                        if feat_value in curr_node.children:
+                            curr_node = curr_node.children[feat_value]
+                        else:
+                            y_pred.append(curr_node.majority_label)
+                            break
+                    case _:
+                        raise TypeError(f"Unknown node type encountered: {type(curr_node)}")
+
+        return np.asarray(self.classes_[y_pred])
+
+    def _id3_algorithm(self, df: pd.DataFrame, feat_indices: set[int]) -> "DecisionTreeNode":
+        """Recursively build the decision tree using the ID3 algorithm.
+
+        Args:
+            df: A pandas.DataFrame containing the training data and target labels for the current node.
+            feat_indices: A set of remaining feature indices available for splitting.
+
+        Returns:
+            A `DecisionTreeNode` representing the root of the constructed subtree
+            (either a `SplitterNode` or `LeafNode`).
+        """
+        y_counts = df["y"].value_counts()
+        most_frequent_y = y_counts.index[0]
+
+        if len(y_counts) == 1 or not feat_indices:
+            return LeafNode(label=most_frequent_y)
+
+        best_feat = max(feat_indices, key=lambda x: self._information_gain(df, x))
+        node = SplitterNode(feat_index=best_feat, majority_label=most_frequent_y)
+
+        for best_feat_value, df_group in df.groupby(by=best_feat):
+            child_node = self._id3_algorithm(df_group, feat_indices - {best_feat})
+            node.children[best_feat_value] = child_node
+
+        all_best_feat_values = self.feature_unique_values_[best_feat]
+        df_best_feat_values = set(df[best_feat])
+        unseen_best_feat_values = all_best_feat_values - df_best_feat_values
+        for best_feat_value in unseen_best_feat_values:
+            node.children[best_feat_value] = LeafNode(label=most_frequent_y)
+
+        return node
+
+    def _information_gain(self, df: pd.DataFrame, feat_index: int) -> float:
+        """Calculate the information gain for a potential feature split.
+
+        Args:
+            df: A pandas.DataFrame containing the current node's data and labels.
+            feat_index: The index of the feature to evaluate.
+
+        Returns:
+            The calculated information gain (reduction in entropy).
+        """
+        total_entropy = self._calc_entropy(df)
+        values = df[feat_index].unique()
+        weighted_entropy = 0.0
+
+        for value in values:
+            subset = df[df[feat_index] == value]
+            weight = len(subset) / len(df)
+            weighted_entropy += weight * self._calc_entropy(subset)
+
+        return total_entropy - weighted_entropy
+
+    def _calc_entropy(self, df: pd.DataFrame) -> float:
+        """Calculate the Shannon entropy of the target labels in the given data.
+
+        Args:
+            df: A pandas.DataFrame containing the target labels in a 'y' column.
+
+        Returns:
+            The calculated entropy value.
+        """
+        value_counts = df["y"].value_counts(normalize=True)
+        entropy = float(-np.sum(value_counts * np.log2(value_counts)))
+        return entropy
+
+
+class DecisionTreeNode:
+    """Base class for nodes in the decision tree."""
+
+    pass
+
+
+class SplitterNode(DecisionTreeNode):
+    """A decision tree node representing a feature split.
+
+    Args:
+        feat_index: The index of the feature used for splitting at this node.
+        majority_label: The integer-encoded majority class label at this node,
+            used as a fallback during prediction.
+
+    Attributes:
+        children: A dictionary mapping feature values to child `DecisionTreeNode` instances.
+    """
+
+    def __init__(self, feat_index: int, majority_label: int) -> None:
+        self.feat_index: int = feat_index
+        self.majority_label: int = majority_label
+        self.children: dict[Any, DecisionTreeNode] = {}
+
+
+class LeafNode(DecisionTreeNode):
+    """A decision tree node representing a terminal leaf.
+
+    Args:
+        label: The integer-encoded predicted class label for this leaf.
+    """
+
+    def __init__(self, label: int) -> None:
+        self.label: int = label

simple_sklearn/classification/_k_neighbors.py

@@ -0,0 +1,220 @@
+"""K-Nearest Neighbors Classification.
+
+This module provides the `KNeighborsClassifier` class.
+"""
+
+import heapq
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.utils.multiclass import type_of_target
+from sklearn.utils.validation import check_is_fitted, validate_data
+from typing_extensions import Self
+
+
+class KNeighborsClassifier(ClassifierMixin, BaseEstimator):  # type: ignore
+    """Perform K-Nearest Neighbors classification.
+
+    The k-nearest neighbors algorithm classifies a new data point based on the
+    majority class among its `n_neighbors` closest neighbors in the training set. It
+    supports uniform voting weight as well as weighting by the inverse of the
+    distance or squared distance.
+
+    Args:
+        n_neighbors: The number of nearest neighbors.
+        weights: The weight function used in prediction. Must be one of "uniform",
+            "distance", or "distance_squared".
+        eps: A small float added to distances to prevent division by zero when
+            calculating weights.
+
+    Attributes:
+        classes_: The unique class labels observed in the training data.
+        fitted_x_: The validated training input data array.
+        fitted_y_: The validated and label-encoded target values array.
+    """
+
+    classes_: NDArray[Any]
+    fitted_x_: NDArray[Any]
+    fitted_y_: NDArray[Any]
+
+    def __init__(self, n_neighbors: int = 5, weights: str = "uniform", eps: float = 1e-9) -> None:
+        super().__init__()
+        self.n_neighbors = n_neighbors
+        self.weights = weights
+        self.eps = eps
+
+    def fit(self, X: Any, y: Any) -> Self:
+        """Fit the k-nearest neighbors classification model.
+
+        Args:
+            X: Training instances to fit the model. Can be an array-like of shape `(n_samples, n_features)`.
+            y: Target values (class labels) for the training instances. Array-like of shape `(n_samples,)`.
+
+        Returns:
+            The fitted instance.
+
+        Raises:
+            ValueError: If `y` is of a continuous type or if any hyperparameters are invalid.
+        """
+        X, y = validate_data(self, X, y)
+        X = np.array(X)
+        self._validate_self_params()
+
+        if type_of_target(y) in ("continuous", "continuous-multioutput"):
+            raise ValueError(f"Unknown label type: {type_of_target(y)}")
+        self.classes_, y = np.unique(y, return_inverse=True)
+
+        self.fitted_x_ = X
+        self.fitted_y_ = y
+
+        return self
+
+    def predict(self, X: Any) -> NDArray[Any]:
+        """Predict class labels for the given input data.
+
+        Args:
+            X: Instances to predict. Can be an array-like of shape `(n_samples, n_features)`.
+
+        Returns:
+            An array of shape `(n_samples,)` containing the predicted class labels for each sample.
+        """
+        check_is_fitted(self)
+        X = validate_data(self, X, reset=False)
+        X = np.array(X)
+
+        decision_scores = self._decision_function(X)
+        return np.asarray(self.classes_[np.argmax(decision_scores, axis=1)])
+
+    def kneighbors(self, X: Any) -> tuple[NDArray[Any], NDArray[Any]]:
+        """Find the k-nearest neighbors of each sample in the given input data.
+
+        Args:
+            X: Instances to evaluate. Can be an array-like of shape `(n_samples, n_features)`.
+
+        Returns:
+            distances: An array of shape `(n_samples, n_neighbors)` representing the
+                Euclidean distances to the nearest points.
+            indices: An array of shape `(n_samples, n_neighbors)` representing the
+                indices of the nearest points in the fitted training data.
+        """
+        check_is_fitted(self)
+        X = validate_data(self, X, reset=False)
+
+        return self._kneighbors(X)
+
+    def _decision_function(self, X: NDArray[Any]) -> NDArray[Any]:
+        """Compute the decision scores for each class based on neighbor votes.
+
+        Calculates the accumulated weights or unweighted counts for each class
+        based on the `self.n_neighbors` nearest neighbors.
+
+        Args:
+            X: Instances to evaluate. An array of shape `(n_samples, n_features)`.
+
+        Returns:
+            An array of shape `(n_samples, n_classes)` containing the decision scores
+                (accumulated votes) for each sample and class.
+        """
+        decision_scores = []
+        for x in X:
+            x_neigh_indices = self._find_kneighbors_indices(x, self.n_neighbors)
+            x_neigh_labels = self.fitted_y_[x_neigh_indices]
+            x_neigh_distances, x_neigh_distances_squared = self._calc_distances(
+                x, X_targets=self.fitted_x_[x_neigh_indices]
+            )
+            if self.weights == "distance":
+                weights = 1 / (x_neigh_distances + self.eps)
+                x_decision_scores = np.bincount(x_neigh_labels, minlength=len(self.classes_), weights=weights)
+            elif self.weights == "distance_squared":
+                weights = 1 / (x_neigh_distances_squared + self.eps)
+                x_decision_scores = np.bincount(x_neigh_labels, minlength=len(self.classes_), weights=weights)
+            else:  # self.weights == 'uniform'
+                x_decision_scores = np.bincount(x_neigh_labels, minlength=len(self.classes_))
+            decision_scores.append(x_decision_scores)
+
+        return np.array(decision_scores)
+
+    def _kneighbors(self, X: NDArray[Any]) -> tuple[NDArray[Any], NDArray[Any]]:
+        """Compute distances and indices of k-nearest neighbors of each sample.
+
+        Args:
+            X: Instances to evaluate. An array of shape `(n_samples, n_features)`.
+
+        Returns:
+            neigh_distances: Array of shape `(n_samples, n_neighbors)` containing Euclidean distances.
+            neigh_indices: Array of shape `(n_samples, n_neighbors)` containing indices in `self.fitted_x_`.
+        """
+        neigh_distances = []
+        neigh_indices = []
+        for x in X:
+            x_neigh_indices = self._find_kneighbors_indices(x, self.n_neighbors)
+            x_neigh_distances, _ = self._calc_distances(x, X_targets=self.fitted_x_[x_neigh_indices])
+            neigh_distances.append(x_neigh_distances)
+            neigh_indices.append(x_neigh_indices)
+
+        return np.array(neigh_distances), np.array(neigh_indices)
+
+    def _find_kneighbors_indices(self, x: NDArray[Any], n_neighbors: int) -> NDArray[Any]:
+        """Find the indices of the nearest neighbors for a single sample.
+
+        Uses a heap queue to efficiently find the `n_neighbors` samples from `self.fitted_x_`
+        with the smallest squared Euclidean distance.
+
+        Args:
+            x: A single instance to evaluate. An array-like of shape `(n_features,)`.
+            n_neighbors: The number of nearest neighbors to find.
+
+        Returns:
+            An array of shape `(n_neighbors,)` containing the indices of the nearest
+            neighbors in the training data.
+        """
+        indices = list(range(self.fitted_x_.shape[0]))
+        _, distances_squared = self._calc_distances(x)
+        neigh_indices = heapq.nsmallest(n_neighbors, indices, key=lambda i: distances_squared[i])
+        return np.array(neigh_indices)
+
+    def _calc_distances(
+        self, x_source: NDArray[Any], X_targets: NDArray[Any] | None = None
+    ) -> tuple[NDArray[Any], NDArray[Any]]:
+        """Calculate the Euclidean and squared Euclidean distances from a source point.
+
+        Args:
+            x_source: The source instance. An array of shape `(n_features,)`.
+            X_targets: The target instances to measure against. If `None`, distances are
+                calculated against all fitted training instances. An array of shape
+                `(n_targets, n_features)`.
+
+        Returns:
+            distances: An array of shape `(n_targets,)` containing Euclidean distances.
+            distances_squared: An array of shape `(n_targets,)` containing squared Euclidean distances.
+        """
+        if X_targets is None:
+            X_targets = self.fitted_x_
+        distances_squared = np.sum((X_targets - x_source) ** 2, axis=1)
+        distances = np.sqrt(distances_squared)
+        return distances, distances_squared
+
+    def _validate_self_params(self) -> None:
+        """Validate the hyperparameters.
+
+        Raises:
+            ValueError: If `n_neighbors` is not a positive integer, if `weights` is not
+                one of the supported string literals, or if `eps` is not a float in the range (0, 1).
+        """
+        if not isinstance(self.n_neighbors, int) or self.n_neighbors < 1:
+            raise ValueError(
+                f"The 'n_neighbors' parameter of KNeighborsClassifier must be an int in the range [1, inf). "
+                f"Got {self.n_neighbors} instead."
+            )
+        if self.weights not in ("distance", "distance_squared", "uniform"):
+            raise ValueError(
+                f"The 'weights' parameter of KNeighborsClassifier must be a str among "
+                f"{{'distance', 'distance_squared', 'uniform'}}. Got '{self.weights}' instead."
+            )
+        if not isinstance(self.eps, float) or not 0 < self.eps < 1:
+            raise ValueError(
+                f"The 'eps' parameter of KNeighborsClassifier must be a float in the range (0.0, 1). "
+                f"Got {self.eps} instead."
+            )