dsr-data-tools 0.0.1__tar.gz

dsr_data_tools-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Scott Roberts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dsr_data_tools-0.0.1/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: dsr-data-tools
+Version: 0.0.1
+Summary: Generic data handling utilities including data splitting and analysis.
+Author-email: Scott Roberts <scottrdeveloper@gmail.com>
+License: MIT
+Keywords: data,splitting,analysis,ml-data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: dsr-utils>=0.0.1
+Requires-Dist: numpy>=2.0.0
+Requires-Dist: pandas>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Dynamic: license-file
+
+# dsr-data-tools
+
+Data analysis and exploration tools for exploratory data analysis (EDA).
+
+## Features
+
+- **Dataset Analysis**: Comprehensive statistical summaries and data quality assessment
+- **Data Exploration**: Tools for understanding data distributions, correlations, and patterns
+- **Quality Metrics**: Missing value detection, data type analysis, and anomaly identification
+
+## Installation
+
+```bash
+pip install dsr-data-tools
+```
+
+## Usage
+
+```python
+import pandas as pd
+from dsr_data_tools import analyze_dataset
+
+# Load your data
+df = pd.read_csv('data.csv')
+
+# Perform comprehensive analysis
+analyze_dataset(df)
+```
+
+## Requirements
+
+- Python >= 3.9
+- pandas
+- numpy
+- dsr-utils
+
+## License
+
+MIT License - see LICENSE file for details

dsr_data_tools-0.0.1/README.md

@@ -0,0 +1,39 @@
+# dsr-data-tools
+
+Data analysis and exploration tools for exploratory data analysis (EDA).
+
+## Features
+
+- **Dataset Analysis**: Comprehensive statistical summaries and data quality assessment
+- **Data Exploration**: Tools for understanding data distributions, correlations, and patterns
+- **Quality Metrics**: Missing value detection, data type analysis, and anomaly identification
+
+## Installation
+
+```bash
+pip install dsr-data-tools
+```
+
+## Usage
+
+```python
+import pandas as pd
+from dsr_data_tools import analyze_dataset
+
+# Load your data
+df = pd.read_csv('data.csv')
+
+# Perform comprehensive analysis
+analyze_dataset(df)
+```
+
+## Requirements
+
+- Python >= 3.9
+- pandas
+- numpy
+- dsr-utils
+
+## License
+
+MIT License - see LICENSE file for details

dsr_data_tools-0.0.1/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dsr-data-tools"
+version = "0.0.1"
+description = "Generic data handling utilities including data splitting and analysis."
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    {name = "Scott Roberts", email="scottrdeveloper@gmail.com"}
+]
+keywords = ["data", "splitting", "analysis", "ml-data"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+    "dsr-utils>=0.0.1",
+    "numpy>=2.0.0",
+    "pandas>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "black>=23.0",
+    "ruff>=0.1.0",
+]
+
+[tool.setuptools]
+packages = ["dsr_data_tools"]
+
+[tool.setuptools.package-dir]
+"" = "src"

dsr_data_tools-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dsr_data_tools-0.0.1/src/dsr_data_tools/__init__.py

@@ -0,0 +1,33 @@
+"""
+dsr_data_tools: Generic data handling utilities for data splitting and analysis.
+"""
+
+from dsr_data_tools.analysis import (
+    DataframeColumn,
+    DataframeInfo,
+    analyze_column_data,
+    analyze_dataset,
+)
+from dsr_data_tools.recommendations import apply_recommendations
+from dsr_data_tools.enums import (
+    RecommendationType,
+    EncodingStrategy,
+    MissingValueStrategy,
+    OutlierStrategy,
+    ImbalanceStrategy,
+)
+
+__all__ = [
+    "DataframeColumn",
+    "DataframeInfo",
+    "analyze_column_data",
+    "analyze_dataset",
+    "apply_recommendations",
+    "RecommendationType",
+    "EncodingStrategy",
+    "MissingValueStrategy",
+    "OutlierStrategy",
+    "ImbalanceStrategy",
+]
+
+__version__ = "0.0.1"

dsr_data_tools-0.0.1/src/dsr_data_tools/analysis.py

@@ -0,0 +1,467 @@
+from __future__ import annotations
+import pandas as pd
+from typing import Type
+from dsr_utils import strings
+from dsr_data_tools.enums import (
+    RecommendationType,
+    EncodingStrategy,
+    MissingValueStrategy,
+    OutlierStrategy,
+    ImbalanceStrategy,
+)
+from dsr_data_tools.recommendations import (
+    Recommendation,
+    NonInformativeRecommendation,
+    MissingValuesRecommendation,
+    EncodingRecommendation,
+    ClassImbalanceRecommendation,
+    OutlierDetectionRecommendation,
+    BooleanClassificationRecommendation,
+    BinningRecommendation,
+)
+
+
+class DataframeColumn:
+    """Represents metadata for a single DataFrame column.
+
+    Stores column name, non-null count, and data type information for analysis
+    and display purposes.
+
+    Attributes:
+        name (str): The column name.
+        non_null_count (int): Number of non-null values in the column.
+        data_type (Type): The pandas data type of the column.
+
+    Example:
+        >>> df = pd.DataFrame({'age': [25, 30, None, 35]})
+        >>> col = DataframeColumn('age', 3, float)
+        >>> col.name
+        'age'
+        >>> col.non_null_count
+        3
+    """
+    @staticmethod
+    def dfc_list_from_df(df: pd.DataFrame) -> list[DataframeColumn]:
+        """Create a list of DataframeColumn objects from a DataFrame.
+
+        Extracts column names, non-null counts, and data types from the DataFrame
+        and creates a DataframeColumn object for each column.
+
+        Args:
+            df (pd.DataFrame): The DataFrame to extract column information from.
+
+        Returns:
+            list[DataframeColumn]: List of DataframeColumn objects, one per column.
+
+        Example:
+            >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
+            >>> columns = DataframeColumn.dfc_list_from_df(df)
+            >>> len(columns)
+            2
+        """
+        df_columns = df.columns.tolist()
+        df_non_null_count = df.count().tolist()
+        df_data_types = df.dtypes.tolist()
+        n = len(df_columns)
+        dfc_list = []
+
+        for c in range(n):
+            dfc = DataframeColumn(df_columns[c],
+                                  df_non_null_count[c],
+                                  df_data_types[c])
+            dfc_list.append(dfc)
+
+        return dfc_list
+
+    def __init__(
+            self,
+            name: str,
+            non_null_count: int,
+            data_type: Type
+    ):
+        self.__name = name
+        self.__non_null_count = non_null_count
+        self.__data_type = data_type
+
+    @property
+    def name(self) -> str:
+        return self.__name
+
+    @property
+    def non_null_count(self) -> int:
+        return self.__non_null_count
+
+    @property
+    def data_type(self) -> Type:
+        return self.__data_type
+
+
+class DataframeInfo:
+    """Stores comprehensive information about a DataFrame's structure and content.
+
+    Provides a summary of DataFrame characteristics including row counts, duplicate
+    detection, and detailed column information. Used for data exploration and
+    quality assessment.
+
+    Attributes:
+        row_count (int): Total number of rows in the DataFrame.
+        duplicate_row_count (int): Number of duplicate rows detected.
+        columns (list[DataframeColumn]): List of column metadata objects.
+
+    Example:
+        >>> df = pd.DataFrame({
+        ...     'name': ['Alice', 'Bob', 'Alice'],
+        ...     'age': [25, 30, 25]
+        ... })
+        >>> df_info = DataframeInfo(df)
+        >>> df_info.row_count
+        3
+        >>> df_info.duplicate_row_count
+        1
+        >>> len(df_info.columns)
+        2
+    """
+
+    def __init__(
+            self,
+            df: pd.DataFrame
+    ):
+        self.__row_count = len(df)
+        self.__duplicate_row_count = df.duplicated().sum()
+        self.__columns = DataframeColumn.dfc_list_from_df(df)
+
+    @property
+    def row_count(self) -> int:
+        return self.__row_count
+
+    @property
+    def duplicate_row_count(self) -> int:
+        return self.__duplicate_row_count
+
+    @property
+    def columns(self) -> list[DataframeColumn]:
+        return self.__columns
+
+    def info(self):
+        """Display formatted summary of DataFrame information.
+
+        Prints row count, duplicate count, and a table showing column names,
+        non-null counts, and data types for all columns.
+
+        Example:
+            >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
+            >>> df_info = DataframeInfo(df)
+            >>> df_info.info()
+            Rows: 2
+            Duplicate rows: 0
+
+            Column         Non-null   Data type
+            name                 2   object
+            age                  2   int64
+        """
+        print(f'Rows: {self.row_count}')
+        print(f'Duplicate rows: {self.duplicate_row_count}')
+        print()
+        col_headers = ['Column', 'Non-null', 'Data type']
+        col_width = [15, 10, 12]
+        print(
+            f'{col_headers[0]:<{col_width[0]}}{col_headers[1]:>{col_width[1]}}   {col_headers[2]:<{col_width[2]}}')
+
+        for c in self.columns:
+            print(
+                f'{c.name:<{col_width[0]}}{c.non_null_count:>{col_width[1]}}   {c.data_type.name:<{col_width[2]}}')
+
+
+def generate_recommendations(
+    df: pd.DataFrame,
+    target_column: str | None = None
+) -> dict[str, dict[str, Recommendation]]:
+    """Generate data preparation recommendations for each column in a DataFrame.
+
+    Analyzes each column and generates appropriate recommendations based on
+    data characteristics (missing values, cardinality, data type, etc.).
+
+    Args:
+        df (pd.DataFrame): The DataFrame to analyze.
+        target_column (str | None): Name of the target column (for imbalance detection).
+            If provided, class imbalance will be analyzed for this column.
+
+    Returns:
+        dict[str, dict[str, Recommendation]]: Nested dictionary mapping column names
+            to recommendation types to Recommendation instances.
+
+    Example:
+        >>> df = pd.DataFrame({
+        ...     'id': range(100),
+        ...     'name': ['Alice'] * 100,
+        ...     'age': [25] * 100 + [30] * 50,
+        ...     'salary': [50000] * 50 + [100000] * 50
+        ... })
+        >>> recs = generate_recommendations(df, target_column='name')
+        >>> recs['id']  # Non-informative (unique count == row count)
+        >>> recs['age']['encoding']  # Binary encoding recommendation
+    """
+    recommendations: dict[str, dict[str, Recommendation]] = {}
+
+    for col_name in df.columns:
+        col_recommendations: dict[str, Recommendation] = {}
+        series = df[col_name]
+
+        # 1. Check for non-informative columns
+        unique_count = series.nunique()
+        total_rows = len(df)
+        is_numeric = pd.api.types.is_numeric_dtype(series)
+
+        # Non-informative: unique count equals total rows (e.g., ID column)
+        if unique_count == total_rows:
+            rec = NonInformativeRecommendation(
+                type=RecommendationType.NON_INFORMATIVE,
+                column_name=col_name,
+                description=f"Column '{col_name}' has unique value for each row.",
+                reason="Unique count equals row count"
+            )
+            col_recommendations['non_informative'] = rec
+            recommendations[col_name] = col_recommendations
+            continue
+
+        # Non-informative: high cardinality object type (> 25% unique values)
+        if not is_numeric and unique_count > total_rows * 0.25:
+            rec = NonInformativeRecommendation(
+                type=RecommendationType.NON_INFORMATIVE,
+                column_name=col_name,
+                description=f"Column '{col_name}' has high cardinality ({unique_count} unique values).",
+                reason="High cardinality object type"
+            )
+            col_recommendations['non_informative'] = rec
+            recommendations[col_name] = col_recommendations
+            continue
+
+        # 2. Check for missing values
+        missing_count = series.isna().sum()
+        if missing_count > 0:
+            missing_percentage = (missing_count / total_rows) * 100
+
+            # Determine strategy based on percentage
+            if missing_percentage < 10:
+                strategy = MissingValueStrategy.DROP_ROWS
+            elif missing_percentage > 50:
+                strategy = MissingValueStrategy.DROP_COLUMN
+            else:
+                strategy = MissingValueStrategy.IMPUTE
+
+            rec = MissingValuesRecommendation(
+                type=RecommendationType.MISSING_VALUES,
+                column_name=col_name,
+                description=f"Column '{col_name}' has {missing_count} missing values ({missing_percentage:.1f}%).",
+                missing_count=missing_count,
+                missing_percentage=missing_percentage,
+                strategy=strategy
+            )
+            col_recommendations['missing_values'] = rec
+
+        # 3. Check for boolean classification (exactly 2 unique numeric values)
+        # Skip target column as it should remain numeric for classifiers
+        if is_numeric and unique_count == 2 and col_name != target_column:
+            values = sorted(series.dropna().unique().tolist())
+            if values == [0.0, 1.0] or values == [0, 1]:
+                rec = BooleanClassificationRecommendation(
+                    type=RecommendationType.BOOLEAN_CLASSIFICATION,
+                    column_name=col_name,
+                    description=f"Column '{col_name}' should be treated as boolean.",
+                    values=values
+                )
+                col_recommendations['boolean_classification'] = rec
+
+        # 4. Check for encoding recommendations (categorical columns)
+        if not is_numeric and col_name != target_column:
+            # Binary categorical: 2 unique values
+            if unique_count == 2:
+                rec = EncodingRecommendation(
+                    type=RecommendationType.ENCODING,
+                    column_name=col_name,
+                    description=f"Column '{col_name}' is binary categorical; recommend LabelEncoder.",
+                    encoder_type=EncodingStrategy.LABEL,
+                    unique_values=unique_count
+                )
+                col_recommendations['encoding'] = rec
+
+            # Multi-class categorical: 3-10 unique values
+            elif 3 <= unique_count <= 10:
+                rec = EncodingRecommendation(
+                    type=RecommendationType.ENCODING,
+                    column_name=col_name,
+                    description=f"Column '{col_name}' is multi-class categorical; recommend OneHotEncoder.",
+                    encoder_type=EncodingStrategy.ONEHOT,
+                    unique_values=unique_count
+                )
+                col_recommendations['encoding'] = rec
+
+        # 5. Check for outliers (numeric columns)
+        if is_numeric:
+            mean_value = series.mean()
+            max_value = series.max()
+            min_value = series.min()
+
+            # Check if max value significantly exceeds mean (potential outliers)
+            if max_value > mean_value * 2:  # Max is more than 2x the mean
+                rec = OutlierDetectionRecommendation(
+                    type=RecommendationType.OUTLIER_DETECTION,
+                    column_name=col_name,
+                    description=f"Column '{col_name}' has potential outliers (max={max_value:.2f}, mean={mean_value:.2f}).",
+                    strategy=OutlierStrategy.SCALING,
+                    max_value=max_value,
+                    mean_value=mean_value
+                )
+                col_recommendations['outlier_detection'] = rec
+
+        # 6. Check for class imbalance (target column)
+        if col_name == target_column and unique_count <= 2:
+            class_counts = series.value_counts()
+            max_class_percentage = (class_counts.max() / total_rows) * 100
+
+            if max_class_percentage > 70:
+                rec = ClassImbalanceRecommendation(
+                    type=RecommendationType.CLASS_IMBALANCE,
+                    column_name=col_name,
+                    description=f"Target variable '{col_name}' shows class imbalance ({max_class_percentage:.1f}% majority class).",
+                    majority_percentage=max_class_percentage,
+                    strategy=ImbalanceStrategy.CLASS_WEIGHT
+                )
+                col_recommendations['class_imbalance'] = rec
+
+        # 7. Suggest binning for numeric columns (e.g., Age)
+        if is_numeric and col_name.lower() in ['age', 'years']:
+            # Use describe() percentiles to suggest bins
+            desc = series.describe()
+            bins = [series.min() - 1, desc['25%'], desc['50%'],
+                    desc['75%'], series.max()]
+            labels = ['Low', 'Medium_Low', 'Medium_High', 'High']
+
+            rec = BinningRecommendation(
+                type=RecommendationType.BINNING,
+                column_name=col_name,
+                description=f"Column '{col_name}' could be binned into {len(labels)} categories.",
+                bins=bins,
+                labels=labels
+            )
+            col_recommendations['binning'] = rec
+
+        if col_recommendations:
+            recommendations[col_name] = col_recommendations
+
+    return recommendations
+
+
+def analyze_column_data(
+    series: pd.Series,
+    dataframe_column: DataframeColumn
+):
+    """Analyze and print detailed statistics for a single DataFrame column.
+
+    Displays column name, data type, null counts, unique values, min/max values.
+    For float columns, shows integer vs non-integer value counts. For object
+    columns, shows numeric vs non-numeric value counts.
+
+    Args:
+        series (pd.Series): The data series to analyze.
+        dataframe_column (DataframeColumn): Metadata about the column.
+
+    Example:
+        >>> df = pd.DataFrame({'price': [10.5, 20.0, 30.99]})
+        >>> col = DataframeColumn('price', 3, float)
+        >>> analyze_column_data(df['price'], col)
+        # Prints detailed statistics
+    """
+    series_length = len(series)
+    is_float_type = (dataframe_column.data_type.name == 'float64')
+    integer_analysis = ''
+
+    if is_float_type:
+        integer_value_count = series.apply(lambda x: x.is_integer()).sum()
+        non_integer_value_count = series_length - integer_value_count
+        integer_analysis = f"""Integer values:     {integer_value_count}
+Non-integer values: {non_integer_value_count}"""
+
+    is_object_data_type = (dataframe_column.data_type.name == 'object')
+    object_analysis = ''
+
+    if is_object_data_type:
+        numeric_value_count = series.str.isnumeric().sum()
+        non_numeric_value_count = series_length - \
+            series.apply(strings.is_float_string).sum()
+
+        object_analysis = f"""Numeric values:     {numeric_value_count}
+Non-numeric values: {non_numeric_value_count}"""
+
+    analysis = f"""
+Column:             {dataframe_column.name}
+Data type:          {dataframe_column.data_type.name}
+Non-null:           {dataframe_column.non_null_count}
+N/A count:          {series.isna().sum()}
+Unique values:      {series.nunique()}
+Min value:          {series.min()}
+Max value:          {series.max()}"""
+
+    print(analysis)
+
+    if is_float_type:
+        print(integer_analysis)
+
+    if is_object_data_type:
+        print(object_analysis)
+
+
+def analyze_dataset(
+    df: pd.DataFrame,
+    target_column: str | None = None,
+    generate_recs: bool = False
+) -> tuple[DataframeInfo, dict[str, dict[str, Recommendation]] | None]:
+    """Perform comprehensive analysis of all columns in a DataFrame.
+
+    Displays overall DataFrame information (row count, duplicates) followed by
+    detailed analysis of each column including data types, null counts, unique
+    values, and type-specific statistics. Optionally generates data preparation
+    recommendations.
+
+    Args:
+        df (pd.DataFrame): The DataFrame to analyze.
+        target_column (str | None): Name of the target column (for recommendation generation).
+        generate_recs (bool): Whether to generate recommendations. Default is False.
+
+    Returns:
+        tuple[DataframeInfo, dict | None]: A tuple containing:
+            - DataframeInfo object with structured DataFrame information
+            - Recommendations dict (or None if generate_recs is False)
+
+    Example:
+        >>> df = pd.DataFrame({
+        ...     'name': ['Alice', 'Bob', 'Charlie'],
+        ...     'age': [25, 30, 35],
+        ...     'salary': [50000.0, 60000.5, 75000.0]
+        ... })
+        >>> info, recs = analyze_dataset(df, generate_recs=True)
+        # Prints comprehensive analysis of all columns and returns recommendations
+    """
+    df_info = DataframeInfo(df)
+    df_info.info()
+
+    n = len(df_info.columns)
+
+    recommendations = None
+    if generate_recs:
+        recommendations = generate_recommendations(df, target_column)
+
+    for c in range(n):
+        col = df_info.columns[c]
+        analyze_column_data(df[col.name], df_info.columns[c])
+
+        # Display recommendations for this column if available
+        if recommendations and col.name in recommendations:
+            col_recs = recommendations[col.name]
+            if col_recs:
+                print("\n  Recommendations:")
+                for rec_type, recommendation in col_recs.items():
+                    recommendation.info()
+                print()
+
+    return df_info, recommendations