smallaxe 0.1.0__tar.gz

smallaxe-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Henok Yemam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

smallaxe-0.1.0/PKG-INFO

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: smallaxe
+Version: 0.1.0
+Summary: A PySpark MLOps library for simplified model training and optimization
+Author: Henok Yemam
+License: MIT
+Project-URL: Homepage, https://github.com/henokyemam/smallaxe
+Project-URL: Repository, https://github.com/henokyemam/smallaxe
+Project-URL: Issues, https://github.com/henokyemam/smallaxe/issues
+Keywords: pyspark,mlops,machine-learning,spark,xgboost,lightgbm,catboost
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.13,>=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyspark<4.0,>=3.3
+Requires-Dist: hyperopt>=0.2.7
+Requires-Dist: plotly>=5.0.0
+Provides-Extra: xgboost
+Requires-Dist: xgboost>=1.7.0; extra == "xgboost"
+Provides-Extra: lightgbm
+Requires-Dist: lightgbm>=3.3.0; extra == "lightgbm"
+Requires-Dist: synapse-ml-lightgbm>=1.0.0; extra == "lightgbm"
+Provides-Extra: catboost
+Requires-Dist: catboost>=1.1.0; extra == "catboost"
+Provides-Extra: all
+Requires-Dist: smallaxe[xgboost]; extra == "all"
+Requires-Dist: smallaxe[lightgbm]; extra == "all"
+Requires-Dist: smallaxe[catboost]; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-spark>=0.6.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Dynamic: license-file
+
+# smallaxe
+
+[![CI](https://github.com/henokyemam/smallaxe/actions/workflows/ci.yml/badge.svg)](https://github.com/henokyemam/smallaxe/actions/workflows/ci.yml)
+
+A PySpark MLOps library that simplifies model training, evaluation, and optimization for PySpark DataFrames.
+
+## Why smallaxe?
+
+PySpark MLlib has a steep learning curve and verbose API. **smallaxe** provides a clean, scikit-learn-like interface for common ML workflows while leveraging the distributed power of Spark.
+
+## Features
+
+- **Simple API** - Train models with familiar `fit()`/`predict()` patterns
+- **Multiple Algorithms** - XGBoost, LightGBM, CatBoost, and Random Forest
+- **Preprocessing Pipeline** - Imputer, Scaler, Encoder with chainable pipelines
+- **Hyperparameter Optimization** - Built-in hyperopt integration with early stopping
+- **Automated Training** - Train all algorithms and compare with one call
+- **Visualization** - Plotly-based charts for model evaluation
+- **Cross-Validation** - Train/test split and k-fold with stratified sampling
+
+## Installation
+
+```bash
+pip install smallaxe
+```
+
+Install with optional algorithm dependencies:
+
+```bash
+pip install smallaxe[xgboost]    # XGBoost support
+pip install smallaxe[lightgbm]   # LightGBM support
+pip install smallaxe[catboost]   # CatBoost support
+pip install smallaxe[all]        # All algorithms
+```
+
+## Quick Start
+
+```python
+from smallaxe.training import Regressors
+from smallaxe.datasets import load_sample_regression
+
+# Load sample data
+df = load_sample_regression(spark)
+
+# Train a model
+model = Regressors.random_forest()
+model.fit(df, label_col='price', exclude_cols=['id'])
+
+# Make predictions
+predictions = model.predict(df)
+```
+
+## Usage Examples
+
+### Training with Cross-Validation
+
+```python
+from smallaxe.training import Classifiers
+
+model = Classifiers.xgboost(task='binary')
+model.fit(
+    df,
+    label_col='churn',
+    validation='kfold',
+    n_folds=5,
+    stratified=True
+)
+
+print(model.validation_scores)
+```
+
+### Preprocessing Pipeline
+
+```python
+from smallaxe.pipeline import Pipeline
+from smallaxe.preprocessing import Imputer, Scaler, Encoder
+from smallaxe.training import Regressors
+
+pipeline = Pipeline([
+    ('imputer', Imputer(numerical_strategy='median')),
+    ('scaler', Scaler(method='standard')),
+    ('encoder', Encoder(method='onehot')),
+    ('model', Regressors.xgboost())
+])
+
+pipeline.fit(
+    df,
+    label_col='target',
+    numerical_cols=['age', 'income'],
+    categorical_cols=['city', 'category']
+)
+
+predictions = pipeline.predict(new_df)
+```
+
+### Hyperparameter Optimization
+
+```python
+from smallaxe.search import optimize
+from hyperopt import hp
+
+param_grid = {
+    'max_depth': hp.choice('max_depth', [3, 5, 7, 10]),
+    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3)
+}
+
+best_model = optimize.run(
+    model=Regressors.xgboost(),
+    dataframe=df,
+    label_col='target',
+    param_grid=param_grid,
+    metric='rmse',
+    max_evals=50
+)
+
+print(best_model.best_params)
+```
+
+### Automated Training
+
+```python
+from smallaxe.auto import AutomatedTraining
+
+auto = AutomatedTraining(model_type='classification', metrics=['f1_score', 'auc_roc'])
+auto.fit(
+    df,
+    label_col='churn',
+    numerical_cols=['tenure', 'monthly_charges'],
+    categorical_cols=['contract'],
+    n_folds=5
+)
+
+# Compare all models
+auto.metrics.show()
+
+# Use best model
+predictions = auto.predict(new_df)
+```
+
+## Supported Algorithms
+
+| Algorithm | Regressor | Classifier | Dependencies |
+|-----------|-----------|------------|--------------|
+| Random Forest | Yes | Yes | None (native PySpark) |
+| XGBoost | Yes | Yes | `smallaxe[xgboost]` |
+| LightGBM | Yes | Yes | `smallaxe[lightgbm]` |
+| CatBoost | Yes | Yes | `smallaxe[catboost]` |
+
+## Requirements
+
+- Python 3.8 - 3.12
+- PySpark 3.3+
+
+## License
+
+MIT License

smallaxe-0.1.0/README.md

@@ -0,0 +1,156 @@
+# smallaxe
+
+[![CI](https://github.com/henokyemam/smallaxe/actions/workflows/ci.yml/badge.svg)](https://github.com/henokyemam/smallaxe/actions/workflows/ci.yml)
+
+A PySpark MLOps library that simplifies model training, evaluation, and optimization for PySpark DataFrames.
+
+## Why smallaxe?
+
+PySpark MLlib has a steep learning curve and verbose API. **smallaxe** provides a clean, scikit-learn-like interface for common ML workflows while leveraging the distributed power of Spark.
+
+## Features
+
+- **Simple API** - Train models with familiar `fit()`/`predict()` patterns
+- **Multiple Algorithms** - XGBoost, LightGBM, CatBoost, and Random Forest
+- **Preprocessing Pipeline** - Imputer, Scaler, Encoder with chainable pipelines
+- **Hyperparameter Optimization** - Built-in hyperopt integration with early stopping
+- **Automated Training** - Train all algorithms and compare with one call
+- **Visualization** - Plotly-based charts for model evaluation
+- **Cross-Validation** - Train/test split and k-fold with stratified sampling
+
+## Installation
+
+```bash
+pip install smallaxe
+```
+
+Install with optional algorithm dependencies:
+
+```bash
+pip install smallaxe[xgboost]    # XGBoost support
+pip install smallaxe[lightgbm]   # LightGBM support
+pip install smallaxe[catboost]   # CatBoost support
+pip install smallaxe[all]        # All algorithms
+```
+
+## Quick Start
+
+```python
+from smallaxe.training import Regressors
+from smallaxe.datasets import load_sample_regression
+
+# Load sample data
+df = load_sample_regression(spark)
+
+# Train a model
+model = Regressors.random_forest()
+model.fit(df, label_col='price', exclude_cols=['id'])
+
+# Make predictions
+predictions = model.predict(df)
+```
+
+## Usage Examples
+
+### Training with Cross-Validation
+
+```python
+from smallaxe.training import Classifiers
+
+model = Classifiers.xgboost(task='binary')
+model.fit(
+    df,
+    label_col='churn',
+    validation='kfold',
+    n_folds=5,
+    stratified=True
+)
+
+print(model.validation_scores)
+```
+
+### Preprocessing Pipeline
+
+```python
+from smallaxe.pipeline import Pipeline
+from smallaxe.preprocessing import Imputer, Scaler, Encoder
+from smallaxe.training import Regressors
+
+pipeline = Pipeline([
+    ('imputer', Imputer(numerical_strategy='median')),
+    ('scaler', Scaler(method='standard')),
+    ('encoder', Encoder(method='onehot')),
+    ('model', Regressors.xgboost())
+])
+
+pipeline.fit(
+    df,
+    label_col='target',
+    numerical_cols=['age', 'income'],
+    categorical_cols=['city', 'category']
+)
+
+predictions = pipeline.predict(new_df)
+```
+
+### Hyperparameter Optimization
+
+```python
+from smallaxe.search import optimize
+from hyperopt import hp
+
+param_grid = {
+    'max_depth': hp.choice('max_depth', [3, 5, 7, 10]),
+    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3)
+}
+
+best_model = optimize.run(
+    model=Regressors.xgboost(),
+    dataframe=df,
+    label_col='target',
+    param_grid=param_grid,
+    metric='rmse',
+    max_evals=50
+)
+
+print(best_model.best_params)
+```
+
+### Automated Training
+
+```python
+from smallaxe.auto import AutomatedTraining
+
+auto = AutomatedTraining(model_type='classification', metrics=['f1_score', 'auc_roc'])
+auto.fit(
+    df,
+    label_col='churn',
+    numerical_cols=['tenure', 'monthly_charges'],
+    categorical_cols=['contract'],
+    n_folds=5
+)
+
+# Compare all models
+auto.metrics.show()
+
+# Use best model
+predictions = auto.predict(new_df)
+```
+
+## Supported Algorithms
+
+| Algorithm | Regressor | Classifier | Dependencies |
+|-----------|-----------|------------|--------------|
+| Random Forest | Yes | Yes | None (native PySpark) |
+| XGBoost | Yes | Yes | `smallaxe[xgboost]` |
+| LightGBM | Yes | Yes | `smallaxe[lightgbm]` |
+| CatBoost | Yes | Yes | `smallaxe[catboost]` |
+
+## Requirements
+
+- Python 3.8 - 3.12
+- PySpark 3.3+
+
+## License
+
+MIT License

smallaxe-0.1.0/pyproject.toml

@@ -0,0 +1,102 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "smallaxe"
+version = "0.1.0"
+description = "A PySpark MLOps library for simplified model training and optimization"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.8,<3.13"
+authors = [
+    {name = "Henok Yemam"}
+]
+keywords = ["pyspark", "mlops", "machine-learning", "spark", "xgboost", "lightgbm", "catboost"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    "pyspark>=3.3,<4.0",
+    "hyperopt>=0.2.7",
+    "plotly>=5.0.0",
+]
+
+[project.optional-dependencies]
+xgboost = [
+    "xgboost>=1.7.0",
+]
+lightgbm = [
+    "lightgbm>=3.3.0",
+    "synapse-ml-lightgbm>=1.0.0",
+]
+catboost = [
+    "catboost>=1.1.0",
+]
+all = [
+    "smallaxe[xgboost]",
+    "smallaxe[lightgbm]",
+    "smallaxe[catboost]",
+]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-spark>=0.6.0",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/henokyemam/smallaxe"
+Repository = "https://github.com/henokyemam/smallaxe"
+Issues = "https://github.com/henokyemam/smallaxe/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["smallaxe*"]
+
+[tool.black]
+line-length = 100
+target-version = ["py38", "py39", "py310", "py311", "py312"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py38"
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+]
+ignore = [
+    "E501",  # line too long (handled by black)
+]
+
+[tool.mypy]
+python_version = "3.8"
+warn_return_any = true
+warn_unused_configs = true
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+addopts = "-v"

smallaxe-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

smallaxe-0.1.0/smallaxe/__init__.py

@@ -0,0 +1,157 @@
+"""
+smallaxe - A PySpark MLOps library for simplified model training and optimization.
+"""
+
+from contextlib import contextmanager
+from typing import Any, Generator, Optional
+
+from smallaxe._config import (
+    VALID_CACHE_STRATEGIES,
+    VALID_VERBOSITY_LEVELS,
+    _config,
+)
+from smallaxe.exceptions import ConfigurationError
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "set_verbosity",
+    "get_verbosity",
+    "verbosity",
+    "set_spark_session",
+    "get_spark_session",
+    "set_seed",
+    "get_seed",
+    "set_cache_strategy",
+    "get_cache_strategy",
+]
+
+
+def set_verbosity(level: str) -> None:
+    """Set the global verbosity level.
+
+    Args:
+        level: Verbosity level. One of 'quiet', 'normal', or 'verbose'.
+            - 'quiet': Only errors, no progress bars or info messages
+            - 'normal': Progress bars and key info (default)
+            - 'verbose': Detailed logging for debugging
+
+    Raises:
+        ConfigurationError: If level is not a valid verbosity level.
+    """
+    if level not in VALID_VERBOSITY_LEVELS:
+        raise ConfigurationError(
+            setting="verbosity",
+            value=level,
+            allowed_values=list(VALID_VERBOSITY_LEVELS),
+        )
+    _config._verbosity = level
+
+
+def get_verbosity() -> str:
+    """Get the current verbosity level.
+
+    Returns:
+        The current verbosity level ('quiet', 'normal', or 'verbose').
+    """
+    return _config._verbosity
+
+
+@contextmanager
+def verbosity(level: str) -> Generator[None, None, None]:
+    """Context manager for temporarily changing verbosity level.
+
+    Args:
+        level: Verbosity level to use within the context.
+
+    Raises:
+        ConfigurationError: If level is not a valid verbosity level.
+
+    Example:
+        >>> with smallaxe.verbosity('quiet'):
+        ...     model.fit(df, label_col='target')  # runs silently
+    """
+    previous_level = get_verbosity()
+    set_verbosity(level)
+    try:
+        yield
+    finally:
+        _config._verbosity = previous_level
+
+
+def set_spark_session(spark: Any) -> None:
+    """Set the Spark session to use.
+
+    Args:
+        spark: A SparkSession instance. If None, smallaxe will attempt
+            to get or create a session when needed.
+    """
+    _config._spark_session = spark
+
+
+def get_spark_session() -> Optional[Any]:
+    """Get the configured Spark session.
+
+    Returns:
+        The configured SparkSession, or None if not set.
+    """
+    return _config._spark_session
+
+
+def set_seed(seed: Optional[int]) -> None:
+    """Set the global random seed for reproducibility.
+
+    This affects all random operations including train/test splits,
+    k-fold cross-validation, and hyperopt sampling.
+
+    Args:
+        seed: Integer seed value, or None to reset to no seed.
+
+    Raises:
+        ConfigurationError: If seed is not an integer or None.
+    """
+    if seed is not None and not isinstance(seed, int):
+        raise ConfigurationError(
+            message=f"Seed must be an integer or None, got {type(seed).__name__}."
+        )
+    _config._seed = seed
+
+
+def get_seed() -> Optional[int]:
+    """Get the current random seed.
+
+    Returns:
+        The current seed value, or None if not set.
+    """
+    return _config._seed
+
+
+def set_cache_strategy(strategy: str) -> None:
+    """Set the caching strategy for PySpark operations.
+
+    Args:
+        strategy: Cache strategy. One of 'auto', 'always', or 'never'.
+            - 'auto': Smart caching - cache after preprocessing, unpersist after training
+            - 'always': Cache at every stage (use for debugging or small datasets)
+            - 'never': No automatic caching (manual control)
+
+    Raises:
+        ConfigurationError: If strategy is not a valid cache strategy.
+    """
+    if strategy not in VALID_CACHE_STRATEGIES:
+        raise ConfigurationError(
+            setting="cache_strategy",
+            value=strategy,
+            allowed_values=list(VALID_CACHE_STRATEGIES),
+        )
+    _config._cache_strategy = strategy
+
+
+def get_cache_strategy() -> str:
+    """Get the current cache strategy.
+
+    Returns:
+        The current cache strategy ('auto', 'always', or 'never').
+    """
+    return _config._cache_strategy

smallaxe-0.1.0/smallaxe/_config.py

@@ -0,0 +1,37 @@
+"""Internal configuration state for smallaxe."""
+
+from typing import Any, Optional
+
+# Valid configuration values
+VALID_VERBOSITY_LEVELS = ("quiet", "normal", "verbose")
+VALID_CACHE_STRATEGIES = ("auto", "always", "never")
+
+# Default configuration values
+DEFAULT_VERBOSITY = "normal"
+DEFAULT_CACHE_STRATEGY = "auto"
+DEFAULT_SEED = None
+
+
+class _Config:
+    """Internal configuration state container.
+
+    This class holds the global configuration state for smallaxe.
+    It should not be accessed directly - use the module-level functions instead.
+    """
+
+    def __init__(self) -> None:
+        self._verbosity: str = DEFAULT_VERBOSITY
+        self._cache_strategy: str = DEFAULT_CACHE_STRATEGY
+        self._seed: Optional[int] = DEFAULT_SEED
+        self._spark_session: Optional[Any] = None
+
+    def reset(self) -> None:
+        """Reset configuration to defaults."""
+        self._verbosity = DEFAULT_VERBOSITY
+        self._cache_strategy = DEFAULT_CACHE_STRATEGY
+        self._seed = DEFAULT_SEED
+        self._spark_session = None
+
+
+# Global configuration instance
+_config = _Config()

smallaxe-0.1.0/smallaxe/auto/__init__.py

@@ -0,0 +1 @@
+"""Auto module - automated training."""

smallaxe-0.1.0/smallaxe/datasets/__init__.py

@@ -0,0 +1,13 @@
+"""Sample datasets for testing and demos."""
+
+from smallaxe.datasets._data import (
+    dataset_info,
+    load_sample_classification,
+    load_sample_regression,
+)
+
+__all__ = [
+    "load_sample_regression",
+    "load_sample_classification",
+    "dataset_info",
+]