adamops 0.1.0__py3-none-any.whl

adamops/data/preprocessors.py

@@ -0,0 +1,227 @@
+"""
+AdamOps Data Preprocessors Module
+
+Provides data cleaning capabilities: missing values, outliers, duplicates, type conversion.
+"""
+
+from typing import Dict, List, Optional, Union
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer, KNNImputer
+from sklearn.experimental import enable_iterative_imputer
+from sklearn.impute import IterativeImputer
+from sklearn.ensemble import IsolationForest
+from adamops.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+# Missing Value Handling
+def handle_missing(
+    df: pd.DataFrame, strategy: str = "mean", columns: Optional[List[str]] = None,
+    fill_value: Optional[any] = None, n_neighbors: int = 5
+) -> pd.DataFrame:
+    """
+    Handle missing values.
+    
+    Args:
+        df: DataFrame to process.
+        strategy: 'drop', 'mean', 'median', 'mode', 'constant', 'ffill', 'bfill', 'knn', 'iterative'
+        columns: Columns to process (None for all).
+        fill_value: Value for 'constant' strategy.
+        n_neighbors: Neighbors for KNN.
+    
+    Returns:
+        Processed DataFrame.
+    """
+    df = df.copy()
+    cols = columns or df.columns.tolist()
+    logger.info(f"Handling missing values with strategy: {strategy}")
+
+    if strategy == "drop":
+        return df.dropna(subset=cols)
+    elif strategy == "ffill":
+        df[cols] = df[cols].ffill()
+    elif strategy == "bfill":
+        df[cols] = df[cols].bfill()
+    elif strategy == "constant":
+        df[cols] = df[cols].fillna(fill_value)
+    elif strategy in ["mean", "median", "most_frequent"]:
+        strat = "most_frequent" if strategy == "mode" else strategy
+        num_cols = [c for c in cols if pd.api.types.is_numeric_dtype(df[c])]
+        if num_cols:
+            imputer = SimpleImputer(strategy=strat)
+            df[num_cols] = imputer.fit_transform(df[num_cols])
+    elif strategy == "mode":
+        for col in cols:
+            if df[col].isna().any():
+                mode_val = df[col].mode().iloc[0] if not df[col].mode().empty else None
+                df[col] = df[col].fillna(mode_val)
+    elif strategy == "knn":
+        num_cols = [c for c in cols if pd.api.types.is_numeric_dtype(df[c])]
+        if num_cols:
+            imputer = KNNImputer(n_neighbors=n_neighbors)
+            df[num_cols] = imputer.fit_transform(df[num_cols])
+    elif strategy == "iterative":
+        num_cols = [c for c in cols if pd.api.types.is_numeric_dtype(df[c])]
+        if num_cols:
+            imputer = IterativeImputer(random_state=42)
+            df[num_cols] = imputer.fit_transform(df[num_cols])
+    
+    return df
+
+
+# Outlier Handling
+def handle_outliers(
+    df: pd.DataFrame, method: str = "iqr", columns: Optional[List[str]] = None,
+    threshold: float = 1.5, action: str = "clip", contamination: float = 0.1
+) -> pd.DataFrame:
+    """
+    Handle outliers.
+    
+    Args:
+        df: DataFrame to process.
+        method: 'iqr', 'zscore', 'isolation_forest'
+        columns: Columns to process (None for numeric).
+        threshold: IQR multiplier or Z-score threshold.
+        action: 'clip', 'drop', 'nan'
+        contamination: For isolation forest.
+    
+    Returns:
+        Processed DataFrame.
+    """
+    df = df.copy()
+    num_cols = columns or df.select_dtypes(include=[np.number]).columns.tolist()
+    logger.info(f"Handling outliers with method: {method}, action: {action}")
+
+    if method == "iqr":
+        for col in num_cols:
+            Q1, Q3 = df[col].quantile(0.25), df[col].quantile(0.75)
+            IQR = Q3 - Q1
+            lower, upper = Q1 - threshold * IQR, Q3 + threshold * IQR
+            mask = (df[col] < lower) | (df[col] > upper)
+            if action == "clip":
+                df[col] = df[col].clip(lower, upper)
+            elif action == "drop":
+                df = df[~mask]
+            elif action == "nan":
+                df.loc[mask, col] = np.nan
+                
+    elif method == "zscore":
+        for col in num_cols:
+            z = np.abs((df[col] - df[col].mean()) / df[col].std())
+            mask = z > threshold
+            if action == "clip":
+                mean, std = df[col].mean(), df[col].std()
+                lower, upper = mean - threshold * std, mean + threshold * std
+                df[col] = df[col].clip(lower, upper)
+            elif action == "drop":
+                df = df[~mask]
+            elif action == "nan":
+                df.loc[mask, col] = np.nan
+                
+    elif method == "isolation_forest":
+        iso = IsolationForest(contamination=contamination, random_state=42)
+        preds = iso.fit_predict(df[num_cols])
+        mask = preds == -1
+        if action == "drop":
+            df = df[~mask]
+        elif action == "nan":
+            df.loc[mask, num_cols] = np.nan
+
+    return df
+
+
+# Duplicate Handling
+def handle_duplicates(
+    df: pd.DataFrame, subset: Optional[List[str]] = None, keep: str = "first"
+) -> pd.DataFrame:
+    """Remove duplicate rows."""
+    before = len(df)
+    df = df.drop_duplicates(subset=subset, keep=keep)
+    logger.info(f"Removed {before - len(df)} duplicates")
+    return df
+
+
+# Type Conversion
+def convert_types(
+    df: pd.DataFrame, type_mapping: Optional[Dict[str, str]] = None,
+    auto_convert: bool = True, datetime_columns: Optional[List[str]] = None
+) -> pd.DataFrame:
+    """
+    Convert column types.
+    
+    Args:
+        df: DataFrame to process.
+        type_mapping: {column: target_type}
+        auto_convert: Auto-detect and convert types.
+        datetime_columns: Columns to parse as datetime.
+    """
+    df = df.copy()
+    
+    if type_mapping:
+        for col, dtype in type_mapping.items():
+            if col in df.columns:
+                try:
+                    df[col] = df[col].astype(dtype)
+                except (ValueError, TypeError) as e:
+                    logger.warning(f"Could not convert {col} to {dtype}: {e}")
+
+    if datetime_columns:
+        for col in datetime_columns:
+            if col in df.columns:
+                df[col] = pd.to_datetime(df[col], errors='coerce')
+
+    if auto_convert:
+        for col in df.columns:
+            if df[col].dtype == 'object':
+                try:
+                    df[col] = pd.to_numeric(df[col], errors='ignore')
+                except:
+                    pass
+    
+    return df
+
+
+# Text Cleaning
+def clean_text(
+    df: pd.DataFrame, columns: Optional[List[str]] = None,
+    lowercase: bool = True, strip: bool = True, remove_special: bool = False
+) -> pd.DataFrame:
+    """Clean text columns."""
+    df = df.copy()
+    str_cols = columns or df.select_dtypes(include=['object']).columns.tolist()
+    
+    for col in str_cols:
+        if strip:
+            df[col] = df[col].str.strip()
+        if lowercase:
+            df[col] = df[col].str.lower()
+        if remove_special:
+            df[col] = df[col].str.replace(r'[^\w\s]', '', regex=True)
+    
+    return df
+
+
+# Full Pipeline
+def preprocess(
+    df: pd.DataFrame, missing_strategy: str = "mean", outlier_method: Optional[str] = None,
+    remove_duplicates: bool = True, convert_types_auto: bool = True
+) -> pd.DataFrame:
+    """Full preprocessing pipeline."""
+    logger.info("Starting preprocessing pipeline")
+    
+    if remove_duplicates:
+        df = handle_duplicates(df)
+    
+    if missing_strategy:
+        df = handle_missing(df, strategy=missing_strategy)
+    
+    if outlier_method:
+        df = handle_outliers(df, method=outlier_method)
+    
+    if convert_types_auto:
+        df = convert_types(df, auto_convert=True)
+    
+    logger.info(f"Preprocessing complete. Shape: {df.shape}")
+    return df

adamops/data/splitters.py

@@ -0,0 +1,218 @@
+"""
+AdamOps Data Splitters Module
+
+Provides data splitting: train/test, train/val/test, time-series, K-Fold, stratified.
+"""
+
+from typing import Iterator, List, Optional, Tuple, Union
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import (
+    train_test_split, KFold, StratifiedKFold, TimeSeriesSplit, GroupKFold
+)
+from adamops.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def split_train_test(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]] = None,
+    test_size: float = 0.2, random_state: int = 42, stratify: bool = False, shuffle: bool = True
+) -> Tuple:
+    """
+    Split data into train and test sets.
+    
+    Args:
+        X: Features.
+        y: Target (optional).
+        test_size: Test set proportion.
+        random_state: Random seed.
+        stratify: Stratify by target.
+        shuffle: Shuffle before splitting.
+    
+    Returns:
+        (X_train, X_test) or (X_train, X_test, y_train, y_test)
+    """
+    stratify_col = y if stratify and y is not None else None
+    
+    if y is not None:
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, test_size=test_size, random_state=random_state, 
+            stratify=stratify_col, shuffle=shuffle
+        )
+        logger.info(f"Split: train={len(X_train)}, test={len(X_test)}")
+        return X_train, X_test, y_train, y_test
+    else:
+        X_train, X_test = train_test_split(
+            X, test_size=test_size, random_state=random_state, shuffle=shuffle
+        )
+        logger.info(f"Split: train={len(X_train)}, test={len(X_test)}")
+        return X_train, X_test
+
+
+def split_train_val_test(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]] = None,
+    train_size: float = 0.7, val_size: float = 0.15, test_size: float = 0.15,
+    random_state: int = 42, stratify: bool = False
+) -> Tuple:
+    """
+    Split data into train, validation, and test sets.
+    
+    Returns:
+        (X_train, X_val, X_test) or (X_train, X_val, X_test, y_train, y_val, y_test)
+    """
+    # Normalize sizes
+    total = train_size + val_size + test_size
+    train_size, val_size, test_size = train_size/total, val_size/total, test_size/total
+    
+    stratify_col = y if stratify and y is not None else None
+    
+    if y is not None:
+        # First split: train+val vs test
+        X_temp, X_test, y_temp, y_test = train_test_split(
+            X, y, test_size=test_size, random_state=random_state, 
+            stratify=stratify_col
+        )
+        # Second split: train vs val
+        val_ratio = val_size / (train_size + val_size)
+        stratify_temp = y_temp if stratify else None
+        X_train, X_val, y_train, y_val = train_test_split(
+            X_temp, y_temp, test_size=val_ratio, random_state=random_state,
+            stratify=stratify_temp
+        )
+        logger.info(f"Split: train={len(X_train)}, val={len(X_val)}, test={len(X_test)}")
+        return X_train, X_val, X_test, y_train, y_val, y_test
+    else:
+        X_temp, X_test = train_test_split(X, test_size=test_size, random_state=random_state)
+        val_ratio = val_size / (train_size + val_size)
+        X_train, X_val = train_test_split(X_temp, test_size=val_ratio, random_state=random_state)
+        return X_train, X_val, X_test
+
+
+def split_timeseries(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]] = None,
+    n_splits: int = 5, test_size: Optional[int] = None, gap: int = 0
+) -> Iterator[Tuple]:
+    """
+    Time series split for temporal data.
+    
+    Args:
+        X: Features.
+        y: Target.
+        n_splits: Number of splits.
+        test_size: Test set size per split.
+        gap: Gap between train and test.
+    
+    Yields:
+        (train_idx, test_idx) tuples.
+    """
+    tscv = TimeSeriesSplit(n_splits=n_splits, test_size=test_size, gap=gap)
+    logger.info(f"Time series split with {n_splits} folds")
+    
+    for train_idx, test_idx in tscv.split(X):
+        yield train_idx, test_idx
+
+
+def split_kfold(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]] = None,
+    n_splits: int = 5, shuffle: bool = True, random_state: int = 42
+) -> Iterator[Tuple]:
+    """
+    K-Fold cross-validation split.
+    
+    Yields:
+        (train_idx, test_idx) tuples.
+    """
+    kf = KFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    logger.info(f"K-Fold split with {n_splits} folds")
+    
+    for train_idx, test_idx in kf.split(X):
+        yield train_idx, test_idx
+
+
+def split_stratified_kfold(
+    X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray],
+    n_splits: int = 5, shuffle: bool = True, random_state: int = 42
+) -> Iterator[Tuple]:
+    """
+    Stratified K-Fold cross-validation split.
+    
+    Preserves class distribution in each fold.
+    
+    Yields:
+        (train_idx, test_idx) tuples.
+    """
+    skf = StratifiedKFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    logger.info(f"Stratified K-Fold split with {n_splits} folds")
+    
+    for train_idx, test_idx in skf.split(X, y):
+        yield train_idx, test_idx
+
+
+def split_group_kfold(
+    X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray],
+    groups: Union[pd.Series, np.ndarray], n_splits: int = 5
+) -> Iterator[Tuple]:
+    """
+    Group K-Fold split. Ensures groups are not split across train/test.
+    
+    Yields:
+        (train_idx, test_idx) tuples.
+    """
+    gkf = GroupKFold(n_splits=n_splits)
+    logger.info(f"Group K-Fold split with {n_splits} folds")
+    
+    for train_idx, test_idx in gkf.split(X, y, groups):
+        yield train_idx, test_idx
+
+
+def get_fold_data(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]],
+    train_idx: np.ndarray, test_idx: np.ndarray
+) -> Tuple:
+    """Get train/test data for a fold."""
+    if isinstance(X, pd.DataFrame):
+        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
+    else:
+        X_train, X_test = X[train_idx], X[test_idx]
+    
+    if y is not None:
+        if isinstance(y, pd.Series):
+            y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
+        else:
+            y_train, y_test = y[train_idx], y[test_idx]
+        return X_train, X_test, y_train, y_test
+    
+    return X_train, X_test
+
+
+def create_cv_splits(
+    X: Union[pd.DataFrame, np.ndarray], y: Optional[Union[pd.Series, np.ndarray]] = None,
+    method: str = "kfold", n_splits: int = 5, **kwargs
+) -> List[Tuple]:
+    """
+    Create cross-validation splits.
+    
+    Args:
+        X: Features.
+        y: Target.
+        method: 'kfold', 'stratified', 'timeseries', 'group'
+        n_splits: Number of folds.
+    
+    Returns:
+        List of (train_idx, test_idx) tuples.
+    """
+    if method == "kfold":
+        return list(split_kfold(X, y, n_splits, **kwargs))
+    elif method == "stratified":
+        if y is None:
+            raise ValueError("y is required for stratified split")
+        return list(split_stratified_kfold(X, y, n_splits, **kwargs))
+    elif method == "timeseries":
+        return list(split_timeseries(X, y, n_splits, **kwargs))
+    elif method == "group":
+        if "groups" not in kwargs:
+            raise ValueError("groups is required for group split")
+        return list(split_group_kfold(X, y, kwargs["groups"], n_splits))
+    else:
+        raise ValueError(f"Unknown split method: {method}")

adamops/data/validators.py

@@ -0,0 +1,148 @@
+"""
+AdamOps Data Validators Module
+
+Provides data validation: type validation, missing value checks, 
+duplicate detection, shape validation, and statistical checks.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+from datetime import datetime
+import numpy as np
+import pandas as pd
+from adamops.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class ValidationIssue:
+    """Represents a validation issue."""
+    severity: str  # 'error', 'warning', 'info'
+    category: str
+    column: Optional[str]
+    message: str
+    details: Optional[Dict] = None
+
+
+@dataclass
+class ColumnStats:
+    """Statistics for a column."""
+    name: str
+    dtype: str
+    count: int
+    missing_count: int
+    missing_pct: float
+    unique_count: int
+    unique_pct: float
+    mean: Optional[float] = None
+    std: Optional[float] = None
+    min: Optional[float] = None
+    max: Optional[float] = None
+
+
+@dataclass
+class ValidationReport:
+    """Complete validation report."""
+    timestamp: str
+    shape: Tuple[int, int]
+    memory_usage: float
+    issues: List[ValidationIssue] = field(default_factory=list)
+    column_stats: Dict[str, ColumnStats] = field(default_factory=dict)
+    duplicate_rows: int = 0
+    passed: bool = True
+
+    def summary(self) -> str:
+        """Generate text summary."""
+        lines = [
+            "=" * 50, "VALIDATION REPORT", "=" * 50,
+            f"Shape: {self.shape[0]} rows x {self.shape[1]} columns",
+            f"Memory: {self.memory_usage:.2f} MB",
+            f"Duplicates: {self.duplicate_rows}",
+            f"Status: {'PASSED' if self.passed else 'FAILED'}",
+            f"Issues: {len(self.issues)}", "=" * 50
+        ]
+        for issue in self.issues:
+            col = f"[{issue.column}] " if issue.column else ""
+            lines.append(f"[{issue.severity.upper()}] {col}{issue.message}")
+        return "\n".join(lines)
+
+
+class DataValidator:
+    """Data validator for DataFrames."""
+    
+    def __init__(self, missing_threshold: float = 0.5, unique_threshold: float = 0.95):
+        self.missing_threshold = missing_threshold
+        self.unique_threshold = unique_threshold
+
+    def validate(self, df: pd.DataFrame, schema: Optional[Dict] = None,
+                 required_columns: Optional[List[str]] = None) -> ValidationReport:
+        """Validate a DataFrame."""
+        report = ValidationReport(
+            timestamp=datetime.now().isoformat(),
+            shape=df.shape,
+            memory_usage=df.memory_usage(deep=True).sum() / 1024**2,
+        )
+        
+        # Check required columns
+        if required_columns:
+            missing = set(required_columns) - set(df.columns)
+            for col in missing:
+                report.issues.append(ValidationIssue("error", "schema", col, f"Missing: {col}"))
+
+        # Check duplicates
+        dups = df.duplicated().sum()
+        report.duplicate_rows = dups
+        if dups > 0:
+            report.issues.append(ValidationIssue("warning", "duplicate", None, f"{dups} duplicates"))
+
+        # Column stats
+        for col in df.columns:
+            series = df[col]
+            missing = series.isna().sum()
+            stats = ColumnStats(
+                name=col, dtype=str(series.dtype), count=len(series),
+                missing_count=missing, missing_pct=100*missing/len(series),
+                unique_count=series.nunique(), unique_pct=100*series.nunique()/len(series),
+            )
+            if pd.api.types.is_numeric_dtype(series):
+                stats.mean, stats.std = series.mean(), series.std()
+                stats.min, stats.max = series.min(), series.max()
+            report.column_stats[col] = stats
+            
+            if stats.missing_pct > self.missing_threshold * 100:
+                report.issues.append(ValidationIssue("warning", "missing", col, 
+                    f"High missing: {stats.missing_pct:.1f}%"))
+
+        report.passed = not any(i.severity == "error" for i in report.issues)
+        return report
+
+
+def validate(df: pd.DataFrame, **kwargs) -> ValidationReport:
+    """Validate a DataFrame."""
+    return DataValidator().validate(df, **kwargs)
+
+def check_missing(df: pd.DataFrame) -> Dict[str, Dict]:
+    """Check missing values."""
+    return {col: {"count": int(df[col].isna().sum()), "pct": 100*df[col].isna().mean()} 
+            for col in df.columns if df[col].isna().any()}
+
+def check_duplicates(df: pd.DataFrame, subset: Optional[List[str]] = None) -> pd.DataFrame:
+    """Get duplicate rows."""
+    return df[df.duplicated(subset=subset, keep=False)]
+
+def check_types(df: pd.DataFrame) -> Dict[str, str]:
+    """Get column types."""
+    return {col: str(dtype) for col, dtype in df.dtypes.items()}
+
+def describe_data(df: pd.DataFrame) -> pd.DataFrame:
+    """Generate data description."""
+    stats = []
+    for col in df.columns:
+        s = df[col]
+        row = {"column": col, "dtype": str(s.dtype), "missing": s.isna().sum(), 
+               "unique": s.nunique()}
+        if pd.api.types.is_numeric_dtype(s):
+            row.update({"mean": s.mean(), "std": s.std(), "min": s.min(), "max": s.max()})
+        stats.append(row)
+    return pd.DataFrame(stats)

adamops/deployment/__init__.py

@@ -0,0 +1,21 @@
+"""
+AdamOps Deployment Module
+
+Provides model deployment capabilities:
+- exporters: Export models to ONNX, PMML, TFLite, CoreML
+- api: Create FastAPI/Flask/Streamlit APIs
+- containerize: Docker and Kubernetes deployment
+- cloud: AWS, GCP, Azure deployment
+"""
+
+from adamops.deployment import exporters
+from adamops.deployment import api
+from adamops.deployment import containerize
+from adamops.deployment import cloud
+
+__all__ = [
+    "exporters",
+    "api",
+    "containerize",
+    "cloud",
+]