adamops 0.1.0__py3-none-any.whl

adamops/__init__.py

@@ -0,0 +1,40 @@
+"""
+AdamOps - A comprehensive MLOps library for end-to-end machine learning workflows.
+
+AdamOps provides tools for:
+- Data loading, validation, cleaning, and feature engineering
+- Model training, registry, and ensemble methods
+- AutoML with hyperparameter tuning
+- Model evaluation and explainability
+- Deployment to various platforms
+- Monitoring and drift detection
+- Pipeline orchestration
+
+Author: AdamOps Team
+Version: 0.1.0
+"""
+
+__version__ = "0.1.0"
+__author__ = "AdamOps Team"
+
+# Import core modules for easy access
+from adamops.data import loaders, validators, preprocessors, splitters
+from adamops.models import modelops, registry, ensembles, automl
+from adamops.evaluation import metrics
+from adamops.utils import config, logging as adamops_logging, helpers
+
+__all__ = [
+    "loaders",
+    "validators", 
+    "preprocessors",
+    "splitters",
+    "modelops",
+    "registry",
+    "ensembles",
+    "automl",
+    "metrics",
+    "config",
+    "adamops_logging",
+    "helpers",
+    "__version__",
+]

adamops/cli.py

@@ -0,0 +1,163 @@
+"""
+AdamOps CLI Module
+
+Command-line interface for AdamOps.
+"""
+
+import sys
+from pathlib import Path
+from typing import Optional
+
+try:
+    import click
+    from rich.console import Console
+    from rich.table import Table
+    CLICK_AVAILABLE = True
+except ImportError:
+    CLICK_AVAILABLE = False
+
+if CLICK_AVAILABLE:
+    console = Console()
+    
+    @click.group()
+    @click.version_option(version="0.1.0", prog_name="adamops")
+    def main():
+        """AdamOps - MLOps made simple."""
+        pass
+    
+    @main.command()
+    @click.option("--data", "-d", required=True, help="Path to data file")
+    @click.option("--target", "-t", required=True, help="Target column name")
+    @click.option("--algorithm", "-a", default="auto", help="Algorithm to use")
+    @click.option("--task", default="auto", help="Task type: classification, regression, auto")
+    @click.option("--output", "-o", default="model.joblib", help="Output model path")
+    def train(data: str, target: str, algorithm: str, task: str, output: str):
+        """Train a model."""
+        console.print(f"[bold blue]Loading data from {data}...[/]")
+        
+        from adamops.data.loaders import load_auto
+        from adamops.models.automl import quick_run
+        from adamops.models.modelops import train as train_model
+        from adamops.deployment.exporters import export_joblib
+        
+        df = load_auto(data)
+        X = df.drop(columns=[target])
+        y = df[target]
+        
+        console.print(f"[bold blue]Training {algorithm} model...[/]")
+        
+        if algorithm == "auto":
+            model = quick_run(X, y, task)
+        else:
+            model = train_model(X, y, task, algorithm)
+        
+        export_joblib(model, output)
+        console.print(f"[bold green]Model saved to {output}[/]")
+    
+    @main.command()
+    @click.option("--model", "-m", required=True, help="Path to model file")
+    @click.option("--data", "-d", required=True, help="Path to test data")
+    @click.option("--target", "-t", required=True, help="Target column name")
+    def evaluate(model: str, data: str, target: str):
+        """Evaluate a model."""
+        from adamops.deployment.exporters import load_model
+        from adamops.data.loaders import load_auto
+        from adamops.evaluation.metrics import evaluate as eval_metrics
+        
+        console.print(f"[bold blue]Loading model and data...[/]")
+        
+        model_obj = load_model(model)
+        df = load_auto(data)
+        X = df.drop(columns=[target])
+        y = df[target]
+        
+        y_pred = model_obj.predict(X)
+        metrics = eval_metrics(y, y_pred)
+        
+        table = Table(title="Evaluation Results")
+        table.add_column("Metric", style="cyan")
+        table.add_column("Value", style="green")
+        
+        for name, value in metrics.items():
+            if isinstance(value, float):
+                table.add_row(name, f"{value:.4f}")
+            else:
+                table.add_row(name, str(value))
+        
+        console.print(table)
+    
+    @main.command()
+    @click.option("--model", "-m", required=True, help="Path to model file")
+    @click.option("--type", "deploy_type", default="api", help="Deployment type: api, docker")
+    @click.option("--port", "-p", default=8000, help="API port")
+    @click.option("--output", "-o", default="./deploy", help="Output directory for docker")
+    def deploy(model: str, deploy_type: str, port: int, output: str):
+        """Deploy a model."""
+        from adamops.deployment.exporters import load_model
+        
+        model_obj = load_model(model)
+        
+        if deploy_type == "api":
+            from adamops.deployment.api import run_api
+            console.print(f"[bold blue]Starting API on port {port}...[/]")
+            run_api(model_obj, port=port)
+        
+        elif deploy_type == "docker":
+            from adamops.deployment.containerize import containerize
+            console.print(f"[bold blue]Creating Docker deployment...[/]")
+            result = containerize(model, output)
+            console.print(f"[bold green]Files created in {output}[/]")
+            for name, path in result.items():
+                console.print(f"  - {name}: {path}")
+    
+    @main.command()
+    @click.option("--data", "-d", required=True, help="Path to data file")
+    def validate(data: str):
+        """Validate a data file."""
+        from adamops.data.loaders import load_auto
+        from adamops.data.validators import validate as validate_data
+        
+        console.print(f"[bold blue]Validating {data}...[/]")
+        
+        df = load_auto(data)
+        report = validate_data(df)
+        
+        console.print(report.summary())
+    
+    @main.command()
+    @click.argument("workflow_name")
+    def run_workflow(workflow_name: str):
+        """Run a predefined workflow."""
+        from adamops.pipelines.workflows import create_ml_pipeline
+        
+        console.print(f"[bold blue]Running workflow: {workflow_name}[/]")
+        
+        workflow = create_ml_pipeline(workflow_name)
+        result = workflow.run()
+        
+        console.print(f"[bold green]Workflow completed![/]")
+        console.print(workflow.get_status())
+    
+    @main.command()
+    def info():
+        """Show AdamOps information."""
+        from adamops import __version__
+        
+        console.print("[bold blue]AdamOps - MLOps Made Simple[/]")
+        console.print(f"Version: {__version__}")
+        console.print()
+        console.print("Available commands:")
+        console.print("  train     - Train a model")
+        console.print("  evaluate  - Evaluate a model")
+        console.print("  deploy    - Deploy a model")
+        console.print("  validate  - Validate data")
+        console.print("  run-workflow - Run a workflow")
+
+else:
+    def main():
+        print("CLI requires click and rich. Install with: pip install click rich")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

adamops/data/__init__.py

@@ -0,0 +1,24 @@
+"""
+AdamOps Data Module
+
+Provides comprehensive data handling capabilities:
+- loaders: Load data from various sources (CSV, Excel, JSON, SQL, API, compressed files)
+- validators: Validate data types, missing values, duplicates, shapes, and statistics
+- preprocessors: Clean data (handle missing values, outliers, duplicates, type conversion)
+- feature_engineering: Encode, scale, and generate features
+- splitters: Split data for training and evaluation
+"""
+
+from adamops.data import loaders
+from adamops.data import validators
+from adamops.data import preprocessors
+from adamops.data import feature_engineering
+from adamops.data import splitters
+
+__all__ = [
+    "loaders",
+    "validators",
+    "preprocessors",
+    "feature_engineering",
+    "splitters",
+]

adamops/data/feature_engineering.py

@@ -0,0 +1,284 @@
+"""
+AdamOps Feature Engineering Module
+
+Provides encoding, scaling, feature selection, and auto feature generation.
+"""
+
+from typing import Dict, List, Optional, Tuple, Union
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import (
+    OneHotEncoder, LabelEncoder, OrdinalEncoder, StandardScaler, 
+    MinMaxScaler, RobustScaler, MaxAbsScaler, PolynomialFeatures
+)
+from sklearn.feature_selection import (
+    VarianceThreshold, SelectKBest, mutual_info_classif, mutual_info_regression, RFE
+)
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+from adamops.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+# =============================================================================
+# Encoding
+# =============================================================================
+
+def encode_onehot(
+    df: pd.DataFrame, columns: List[str], drop_first: bool = False,
+    handle_unknown: str = "ignore"
+) -> pd.DataFrame:
+    """One-hot encode categorical columns."""
+    df = df.copy()
+    for col in columns:
+        dummies = pd.get_dummies(df[col], prefix=col, drop_first=drop_first)
+        df = pd.concat([df.drop(col, axis=1), dummies], axis=1)
+    logger.info(f"One-hot encoded {len(columns)} columns")
+    return df
+
+
+def encode_label(df: pd.DataFrame, columns: List[str]) -> Tuple[pd.DataFrame, Dict]:
+    """Label encode categorical columns. Returns df and encoders dict."""
+    df = df.copy()
+    encoders = {}
+    for col in columns:
+        le = LabelEncoder()
+        df[col] = le.fit_transform(df[col].astype(str))
+        encoders[col] = le
+    logger.info(f"Label encoded {len(columns)} columns")
+    return df, encoders
+
+
+def encode_ordinal(
+    df: pd.DataFrame, columns: List[str], 
+    categories: Optional[Dict[str, List]] = None
+) -> pd.DataFrame:
+    """Ordinal encode columns with optional category order."""
+    df = df.copy()
+    for col in columns:
+        if categories and col in categories:
+            cat_map = {v: i for i, v in enumerate(categories[col])}
+            df[col] = df[col].map(cat_map)
+        else:
+            df[col] = pd.Categorical(df[col]).codes
+    return df
+
+
+def encode_target(
+    df: pd.DataFrame, columns: List[str], target: str, smoothing: float = 1.0
+) -> pd.DataFrame:
+    """Target encode categorical columns."""
+    df = df.copy()
+    global_mean = df[target].mean()
+    
+    for col in columns:
+        agg = df.groupby(col)[target].agg(['mean', 'count'])
+        smooth = (agg['count'] * agg['mean'] + smoothing * global_mean) / (agg['count'] + smoothing)
+        df[col + '_target'] = df[col].map(smooth)
+        df = df.drop(col, axis=1)
+    
+    return df
+
+
+def encode(
+    df: pd.DataFrame, columns: List[str], method: str = "onehot", **kwargs
+) -> pd.DataFrame:
+    """Encode categorical columns with specified method."""
+    if method == "onehot":
+        return encode_onehot(df, columns, **kwargs)
+    elif method == "label":
+        return encode_label(df, columns, **kwargs)[0]
+    elif method == "ordinal":
+        return encode_ordinal(df, columns, **kwargs)
+    elif method == "target" and "target" in kwargs:
+        return encode_target(df, columns, kwargs["target"])
+    else:
+        raise ValueError(f"Unknown encoding method: {method}")
+
+
+# =============================================================================
+# Scaling
+# =============================================================================
+
+def scale_standard(df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:
+    """Standardize features (zero mean, unit variance)."""
+    df = df.copy()
+    cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+    scaler = StandardScaler()
+    df[cols] = scaler.fit_transform(df[cols])
+    return df
+
+
+def scale_minmax(df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:
+    """Scale features to [0, 1] range."""
+    df = df.copy()
+    cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+    scaler = MinMaxScaler()
+    df[cols] = scaler.fit_transform(df[cols])
+    return df
+
+
+def scale_robust(df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:
+    """Scale with median and IQR (robust to outliers)."""
+    df = df.copy()
+    cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+    scaler = RobustScaler()
+    df[cols] = scaler.fit_transform(df[cols])
+    return df
+
+
+def scale(
+    df: pd.DataFrame, method: str = "standard", columns: Optional[List[str]] = None
+) -> pd.DataFrame:
+    """Scale numeric columns with specified method."""
+    if method == "standard":
+        return scale_standard(df, columns)
+    elif method == "minmax":
+        return scale_minmax(df, columns)
+    elif method == "robust":
+        return scale_robust(df, columns)
+    elif method == "maxabs":
+        df = df.copy()
+        cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+        df[cols] = MaxAbsScaler().fit_transform(df[cols])
+        return df
+    else:
+        raise ValueError(f"Unknown scaling method: {method}")
+
+
+# =============================================================================
+# Feature Selection
+# =============================================================================
+
+def select_by_variance(
+    df: pd.DataFrame, threshold: float = 0.0, columns: Optional[List[str]] = None
+) -> pd.DataFrame:
+    """Remove low variance features."""
+    cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+    selector = VarianceThreshold(threshold=threshold)
+    selected = selector.fit_transform(df[cols])
+    selected_cols = [cols[i] for i in selector.get_support(indices=True)]
+    df_result = df.drop(cols, axis=1)
+    df_result[selected_cols] = selected
+    logger.info(f"Selected {len(selected_cols)}/{len(cols)} features by variance")
+    return df_result
+
+
+def select_by_correlation(
+    df: pd.DataFrame, threshold: float = 0.9, target: Optional[str] = None
+) -> pd.DataFrame:
+    """Remove highly correlated features."""
+    df = df.copy()
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    if target and target in num_cols:
+        num_cols.remove(target)
+    
+    corr = df[num_cols].corr().abs()
+    upper = corr.where(np.triu(np.ones(corr.shape), k=1).astype(bool))
+    to_drop = [c for c in upper.columns if any(upper[c] > threshold)]
+    
+    logger.info(f"Dropping {len(to_drop)} highly correlated features")
+    return df.drop(to_drop, axis=1)
+
+
+def select_by_importance(
+    df: pd.DataFrame, target: str, n_features: int = 10, task: str = "classification"
+) -> pd.DataFrame:
+    """Select features by tree-based importance."""
+    X = df.drop(target, axis=1).select_dtypes(include=[np.number])
+    y = df[target]
+    
+    model = RandomForestClassifier(n_estimators=50, random_state=42) if task == "classification" \
+        else RandomForestRegressor(n_estimators=50, random_state=42)
+    model.fit(X, y)
+    
+    importance = pd.Series(model.feature_importances_, index=X.columns).sort_values(ascending=False)
+    top_features = importance.head(n_features).index.tolist()
+    
+    logger.info(f"Selected top {n_features} features by importance")
+    return df[[target] + top_features]
+
+
+def select_features(
+    df: pd.DataFrame, target: str, method: str = "importance", n_features: int = 10, **kwargs
+) -> pd.DataFrame:
+    """Select features using specified method."""
+    if method == "variance":
+        return select_by_variance(df, **kwargs)
+    elif method == "correlation":
+        return select_by_correlation(df, target=target, **kwargs)
+    elif method == "importance":
+        return select_by_importance(df, target, n_features, **kwargs)
+    else:
+        raise ValueError(f"Unknown selection method: {method}")
+
+
+# =============================================================================
+# Feature Generation
+# =============================================================================
+
+def generate_polynomial(
+    df: pd.DataFrame, columns: List[str], degree: int = 2, include_bias: bool = False
+) -> pd.DataFrame:
+    """Generate polynomial features."""
+    df = df.copy()
+    poly = PolynomialFeatures(degree=degree, include_bias=include_bias)
+    poly_features = poly.fit_transform(df[columns])
+    poly_names = poly.get_feature_names_out(columns)
+    df_poly = pd.DataFrame(poly_features, columns=poly_names, index=df.index)
+    return pd.concat([df.drop(columns, axis=1), df_poly], axis=1)
+
+
+def generate_interactions(
+    df: pd.DataFrame, columns: List[str], operations: List[str] = ["multiply"]
+) -> pd.DataFrame:
+    """Generate interaction features between columns."""
+    df = df.copy()
+    for i, col1 in enumerate(columns):
+        for col2 in columns[i+1:]:
+            if "multiply" in operations:
+                df[f"{col1}_x_{col2}"] = df[col1] * df[col2]
+            if "add" in operations:
+                df[f"{col1}_+_{col2}"] = df[col1] + df[col2]
+            if "divide" in operations:
+                df[f"{col1}_/_{col2}"] = df[col1] / (df[col2] + 1e-8)
+    return df
+
+
+def generate_datetime_features(df: pd.DataFrame, column: str) -> pd.DataFrame:
+    """Extract datetime features from a column."""
+    df = df.copy()
+    dt = pd.to_datetime(df[column])
+    prefix = column
+    df[f"{prefix}_year"] = dt.dt.year
+    df[f"{prefix}_month"] = dt.dt.month
+    df[f"{prefix}_day"] = dt.dt.day
+    df[f"{prefix}_dayofweek"] = dt.dt.dayofweek
+    df[f"{prefix}_hour"] = dt.dt.hour
+    df[f"{prefix}_is_weekend"] = dt.dt.dayofweek.isin([5, 6]).astype(int)
+    return df
+
+
+def auto_feature_engineering(
+    df: pd.DataFrame, target: Optional[str] = None,
+    polynomial: bool = False, interactions: bool = False, datetime_cols: Optional[List[str]] = None
+) -> pd.DataFrame:
+    """Automatic feature engineering pipeline."""
+    logger.info("Running auto feature engineering")
+    
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    if target and target in num_cols:
+        num_cols.remove(target)
+    
+    if datetime_cols:
+        for col in datetime_cols:
+            df = generate_datetime_features(df, col)
+    
+    if polynomial and len(num_cols) <= 5:
+        df = generate_polynomial(df, num_cols[:5], degree=2)
+    
+    if interactions and len(num_cols) >= 2:
+        df = generate_interactions(df, num_cols[:4])
+    
+    logger.info(f"Feature engineering complete. New shape: {df.shape}")
+    return df