dipencsv 0.1.0__py3-none-any.whl

dipencsv/__init__.py

@@ -0,0 +1,8 @@
+# dipencsv/__init__.py
+
+from dipencsv.core import Data
+
+__version__ = "0.1.0"
+__author__ = "Dipendra"
+
+__all__ = ["Data"]

dipencsv/analytics.py

@@ -0,0 +1,70 @@
+# dipencsv/analytics.py
+
+import pandas as pd
+from dipencsv.errors import format_error, format_success
+
+def describe(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_df = df.select_dtypes(include="number")
+    if numeric_df.empty:
+        print("⚠️  No numeric columns found")
+        return pd.DataFrame()
+    return numeric_df.describe().round(2)
+
+
+def value_counts(df: pd.DataFrame, col: str) -> pd.Series:
+    return df[col].value_counts()
+
+
+def correlation_matrix(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_df = df.select_dtypes(include="number")
+    if numeric_df.empty:
+        print("⚠️  No numeric columns found")
+        return pd.DataFrame()
+    return numeric_df.corr().round(2)
+
+
+def outliers(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    q1 = df[col].quantile(0.25)
+    q3 = df[col].quantile(0.75)
+    iqr = q3 - q1
+    lower = q1 - 1.5 * iqr
+    upper = q3 + 1.5 * iqr
+    result = df[(df[col] < lower) | (df[col] > upper)]
+    print(f"📊 Outliers in '{col}': {len(result)} rows")
+    return result
+
+
+def distribution(df: pd.DataFrame, col: str) -> dict:
+    return {
+        "mean"  : round(float(df[col].mean()), 2),
+        "median": round(float(df[col].median()), 2),
+        "std"   : round(float(df[col].std()), 2),
+        "min"   : round(float(df[col].min()), 2),
+        "max"   : round(float(df[col].max()), 2),
+        "skew"  : round(float(df[col].skew()), 2),
+    }
+def sort(df: pd.DataFrame, col: str, asc: bool = True) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    df = df.sort_values(by=col, ascending=asc).reset_index(drop=True)
+    direction = "ascending" if asc else "descending"
+    print(format_success(f"Sorted by '{col}' ({direction})"))
+    return df
+
+
+def first(df: pd.DataFrame, n: int = 5) -> pd.DataFrame:
+    return df.head(n)
+
+
+def last(df: pd.DataFrame, n: int = 5) -> pd.DataFrame:
+    return df.tail(n)
+
+
+def find(df: pd.DataFrame, col: str, value) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return pd.DataFrame()
+    result = df[df[col] == value]
+    print(format_success(f"Found {len(result)} rows where '{col}' = '{value}'"))
+    return result

dipencsv/cleaner.py

@@ -0,0 +1,154 @@
+# dipencsv/cleaner.py
+
+import pandas as pd
+
+from dipencsv.errors import format_error, format_warning, format_success
+
+
+def clean(df: pd.DataFrame, strategy: str = "safe") -> pd.DataFrame:
+    df = df.copy()
+
+    # fix column names
+    df = _fix_columns(df)
+
+    # remove duplicates
+    before = len(df)
+    df = df.drop_duplicates()
+    removed = before - len(df)
+    if removed > 0:
+        print(format_warning(f"Removed {removed} duplicate rows"))
+
+    # handle missing values
+    df = _handle_missing(df, strategy)
+
+    # convert text numbers to numeric
+    df = _fix_numeric(df)
+
+    print(format_success("Cleaning done"))
+    return df
+
+
+def _fix_columns(df: pd.DataFrame) -> pd.DataFrame:
+    df.columns = (
+        df.columns
+        .str.strip()
+        .str.lower()
+        .str.replace(" ", "_")
+        .str.replace(r"[^\w]", "", regex=True)
+    )
+    return df
+
+
+def _handle_missing(df: pd.DataFrame, strategy: str) -> pd.DataFrame:
+    if strategy == "safe":
+        for col in df.columns:
+            if df[col].isnull().sum() == 0:
+                continue
+            if df[col].dtype in ["float64", "int64"]:
+                df[col] = df[col].fillna(df[col].median())
+            else:
+                mode = df[col].mode()
+                if not mode.empty:
+                    df[col] = df[col].fillna(mode[0])
+    elif strategy == "aggressive":
+        before = len(df)
+        df = df.dropna()
+        print(format_warning(f"Dropped {before - len(df)} rows with missing values"))
+    return df
+
+
+def _fix_numeric(df: pd.DataFrame) -> pd.DataFrame:
+    for col in df.select_dtypes(include=["object", "str"]).columns:
+        converted = pd.to_numeric(df[col], errors="coerce")
+        if converted.notna().sum() > len(df) * 0.7:
+            df[col] = converted
+    return df
+
+def drop(df: pd.DataFrame, cols) -> pd.DataFrame:
+    if isinstance(cols, str):
+        cols = [cols]
+    missing = [c for c in cols if c not in df.columns]
+    if missing:
+        print(format_error(f"Columns not found: {missing}"))
+        return df
+    df = df.drop(columns=cols)
+    print(format_success(f"Dropped columns: {cols}"))
+    return df
+
+
+def rename(df: pd.DataFrame, old: str, new: str) -> pd.DataFrame:
+    if old not in df.columns:
+        print(format_error(f"Column '{old}' not found"))
+        return df
+    df = df.rename(columns={old: new})
+    print(format_success(f"Renamed '{old}' → '{new}'"))
+    return df
+
+
+def fill(df: pd.DataFrame, col: str, value) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    df[col] = df[col].fillna(value)
+    print(format_success(f"Filled missing values in '{col}' with {value}"))
+    return df
+
+def encode(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    unique_vals = df[col].unique()
+    mapping = {val: idx for idx, val in enumerate(unique_vals)}
+    df[col] = df[col].map(mapping)
+    print(format_success(f"Encoded '{col}': {mapping}"))
+    return df
+
+
+def normalize(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    min_val = df[col].min()
+    max_val = df[col].max()
+    if max_val == min_val:
+        print(format_error(f"Cannot normalize '{col}' — all values are the same"))
+        return df
+    df[col] = (df[col] - min_val) / (max_val - min_val)
+    print(format_success(f"Normalized '{col}' to range [0, 1]"))
+    return df
+
+
+def standardize(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    mean = df[col].mean()
+    std = df[col].std()
+    if std == 0:
+        print(format_error(f"Cannot standardize '{col}' — std is 0"))
+        return df
+    df[col] = (df[col] - mean) / std
+    print(format_success(f"Standardized '{col}' — mean=0, std=1"))
+    return df
+
+def remove_outliers(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    if col not in df.columns:
+        print(format_error(f"Column '{col}' not found"))
+        return df
+    q1 = df[col].quantile(0.25)
+    q3 = df[col].quantile(0.75)
+    iqr = q3 - q1
+    lower = q1 - 1.5 * iqr
+    upper = q3 + 1.5 * iqr
+    before = len(df)
+    df = df[(df[col] >= lower) & (df[col] <= upper)]
+    removed = before - len(df)
+    print(format_success(f"Removed {removed} outliers from '{col}'"))
+    return df
+
+
+def split(df: pd.DataFrame, test_size: float = 0.2, random_state: int = 42):
+    from sklearn.model_selection import train_test_split
+    train, test = train_test_split(df, test_size=test_size, random_state=random_state)
+    print(format_success(f"Split done — train: {len(train)} rows, test: {len(test)} rows"))
+    return train, test

dipencsv/core.py

@@ -0,0 +1,226 @@
+# dipencsv/core.py
+
+from dipencsv.loader import load_file, get_file_size_mb
+from dipencsv.engine.pandas_engine import PandasEngine
+from dipencsv.engine.stream_engine import StreamEngine
+from dipencsv.summary import summary, report
+from dipencsv.cleaner import clean
+from dipencsv.analytics import describe, value_counts, correlation_matrix, outliers, distribution
+from dipencsv.intelligence import ask, explain, magic
+from dipencsv.exporter import export
+from dipencsv.cleaner import clean, drop, rename, fill
+from dipencsv.cleaner import clean, drop, rename, fill, encode, normalize, standardize
+from dipencsv.cleaner import clean, drop, rename, fill, encode, normalize, standardize, remove_outliers, split
+from dipencsv.analytics import describe, value_counts, correlation_matrix, outliers, distribution, sort, find, first, last
+
+SIZE_THRESHOLD_MB = 500
+
+
+class Data:
+
+    def __init__(self, filepath: str, stream: bool = False, auto_mode: bool = True):
+        self._filepath = filepath
+        self._engine = self._init_engine(filepath, stream, auto_mode)
+
+    def _init_engine(self, filepath, stream, auto_mode):
+        if stream:
+            print("🌊 Stream mode activated")
+            return StreamEngine(filepath)
+
+        if auto_mode:
+            size = get_file_size_mb(filepath)
+            if size > SIZE_THRESHOLD_MB:
+                print(f"🌊 File is {size:.1f}MB — auto switching to stream mode")
+                return StreamEngine(filepath)
+            else:
+                print(f"🐼 File is {size:.1f}MB — using pandas mode")
+                df = load_file(filepath)
+                return PandasEngine(df)
+
+        df = load_file(filepath)
+        return PandasEngine(df)
+
+    # --- analytics ---
+    def mean(self, col: str) -> float:
+        return self._engine.mean(col)
+
+    def median(self, col: str) -> float:
+        return self._engine.median(col)
+
+    def max(self, col: str) -> float:
+        return self._engine.max(col)
+
+    def min(self, col: str) -> float:
+        return self._engine.min(col)
+
+    def count(self, col: str) -> int:
+        return self._engine.count(col)
+
+    def group_mean(self, group_col: str, value_col: str) -> dict:
+        return self._engine.group_mean(group_col, value_col)
+
+    def correlation(self, col1: str, col2: str) -> float:
+        return self._engine.correlation(col1, col2)
+
+    def filter(self, condition: str):
+        return self._engine.filter(condition)
+
+    def top_n(self, col: str, n: int = 10):
+        return self._engine.top_n(col, n)
+
+    def summary(self):
+        self._engine._df if hasattr(self._engine, '_df') else None
+        if hasattr(self._engine, '_df'):
+            summary(self._engine._df)
+        else:
+            print("⚠️  summary() not supported in stream mode")
+
+    def report(self):
+        if hasattr(self._engine, '_df'):
+            report(self._engine._df)
+        else:
+            print("⚠️  report() not supported in stream mode")
+
+    def clean(self, strategy: str = "safe"):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = clean(self._engine._df, strategy)
+        else:
+            print("⚠️  clean() not supported in stream mode")
+
+    def describe(self):
+        if hasattr(self._engine, '_df'):
+            return describe(self._engine._df)
+        else:
+            print("⚠️  describe() not supported in stream mode")
+
+    def value_counts(self, col: str):
+        if hasattr(self._engine, '_df'):
+            return value_counts(self._engine._df, col)
+        else:
+            print("⚠️  value_counts() not supported in stream mode")
+
+    def correlation_matrix(self):
+        if hasattr(self._engine, '_df'):
+            return correlation_matrix(self._engine._df)
+        else:
+            print("⚠️  correlation_matrix() not supported in stream mode")
+
+    def outliers(self, col: str):
+        if hasattr(self._engine, '_df'):
+            return outliers(self._engine._df, col)
+        else:
+            print("⚠️  outliers() not supported in stream mode")
+
+    def distribution(self, col: str):
+        if hasattr(self._engine, '_df'):
+            return distribution(self._engine._df, col)
+        else:
+            print("⚠️  distribution() not supported in stream mode")
+
+    def ask(self, question: str):
+        if hasattr(self._engine, '_df'):
+            return ask(self._engine._df, question)
+        else:
+            print("⚠️  ask() not supported in stream mode")
+
+    def explain(self):
+        if hasattr(self._engine, '_df'):
+            explain(self._engine._df)
+        else:
+            print("⚠️  explain() not supported in stream mode")
+
+    def magic(self):
+        if hasattr(self._engine, '_df'):
+            magic(self._engine._df)
+        else:
+            print("⚠️  magic() not supported in stream mode")
+
+    def export(self, filepath: str):
+        if hasattr(self._engine, '_df'):
+            export(self._engine._df, filepath)
+        else:
+            print("⚠️  export() not supported in stream mode")
+
+    def drop(self, cols):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = drop(self._engine._df, cols)
+        else:
+            print("⚠️  drop() not supported in stream mode")
+
+    def rename(self, old: str, new: str):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = rename(self._engine._df, old, new)
+        else:
+            print("⚠️  rename() not supported in stream mode")
+
+    def fill(self, col: str, value):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = fill(self._engine._df, col, value)
+        else:
+            print("⚠️  fill() not supported in stream mode")
+
+    def encode(self, col: str):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = encode(self._engine._df, col)
+        else:
+            print("⚠️  encode() not supported in stream mode")
+
+    def normalize(self, col: str):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = normalize(self._engine._df, col)
+        else:
+            print("⚠️  normalize() not supported in stream mode")
+
+    def standardize(self, col: str):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = standardize(self._engine._df, col)
+        else:
+            print("⚠️  standardize() not supported in stream mode")
+
+    def remove_outliers(self, col: str):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = remove_outliers(self._engine._df, col)
+        else:
+            print("⚠️  remove_outliers() not supported in stream mode")
+
+    def split(self, test_size: float = 0.2, random_state: int = 42):
+        if hasattr(self._engine, '_df'):
+            return split(self._engine._df, test_size, random_state)
+        else:
+            print("⚠️  split() not supported in stream mode")
+
+    @property
+    def columns(self) -> list:
+        if hasattr(self._engine, '_df'):
+            return list(self._engine._df.columns)
+        return self._engine._columns
+
+    @property
+    def shape(self) -> tuple:
+        if hasattr(self._engine, '_df'):
+            return self._engine._df.shape
+        return None
+
+    def first(self, n: int = 5):
+        if hasattr(self._engine, '_df'):
+            return first(self._engine._df, n)
+        else:
+            print("⚠️  first() not supported in stream mode")
+
+    def last(self, n: int = 5):
+        if hasattr(self._engine, '_df'):
+            return last(self._engine._df, n)
+        else:
+            print("⚠️  last() not supported in stream mode")
+
+    def sort(self, col: str, asc: bool = True):
+        if hasattr(self._engine, '_df'):
+            self._engine._df = sort(self._engine._df, col, asc)
+        else:
+            print("⚠️  sort() not supported in stream mode")
+
+    def find(self, col: str, value):
+        if hasattr(self._engine, '_df'):
+            return find(self._engine._df, col, value)
+        else:
+            print("⚠️  find() not supported in stream mode")

dipencsv/engine/__init__.py

@@ -0,0 +1,6 @@
+# dipencsv/engine/__init__.py
+
+from dipencsv.engine.pandas_engine import PandasEngine
+from dipencsv.engine.stream_engine import StreamEngine
+
+__all__ = ["PandasEngine", "StreamEngine"]

dipencsv/engine/base.py

@@ -0,0 +1,42 @@
+# dipencsv/engine/base.py
+
+from abc import ABC, abstractmethod
+
+
+class BaseEngine(ABC):
+
+    @abstractmethod
+    def mean(self, col: str) -> float:
+        pass
+
+    @abstractmethod
+    def median(self, col: str) -> float:
+        pass
+
+    @abstractmethod
+    def max(self, col: str) -> float:
+        pass
+
+    @abstractmethod
+    def min(self, col: str) -> float:
+        pass
+
+    @abstractmethod
+    def count(self, col: str) -> int:
+        pass
+
+    @abstractmethod
+    def group_mean(self, group_col: str, value_col: str) -> dict:
+        pass
+
+    @abstractmethod
+    def correlation(self, col1: str, col2: str) -> float:
+        pass
+
+    @abstractmethod
+    def filter(self, condition: str):
+        pass
+
+    @abstractmethod
+    def top_n(self, col: str, n: int):
+        pass

dipencsv/engine/pandas_engine.py

@@ -0,0 +1,53 @@
+# dipencsv/engine/pandas_engine.py
+
+import pandas as pd
+from dipencsv.engine.base import BaseEngine
+from dipencsv.errors import suggest_column
+
+
+class PandasEngine(BaseEngine):
+
+    def __init__(self, df: pd.DataFrame):
+        self._df = df
+
+    def mean(self, col: str) -> float:
+        self._check_col(col)
+        return self._df[col].mean()
+
+    def median(self, col: str) -> float:
+        self._check_col(col)
+        return self._df[col].median()
+
+    def max(self, col: str) -> float:
+        self._check_col(col)
+        return self._df[col].max()
+
+    def min(self, col: str) -> float:
+        self._check_col(col)
+        return self._df[col].min()
+
+    def count(self, col: str) -> int:
+        self._check_col(col)
+        return self._df[col].count()
+
+    def group_mean(self, group_col: str, value_col: str) -> dict:
+        self._check_col(group_col)
+        self._check_col(value_col)
+        return self._df.groupby(group_col)[value_col].mean().to_dict()
+
+    def correlation(self, col1: str, col2: str) -> float:
+        self._check_col(col1)
+        self._check_col(col2)
+        return self._df[col1].corr(self._df[col2])
+
+    def filter(self, condition: str) -> pd.DataFrame:
+        return self._df.query(condition)
+
+    def top_n(self, col: str, n: int = 10) -> pd.DataFrame:
+        self._check_col(col)
+        return self._df.nlargest(n, col)
+
+    def _check_col(self, col: str):
+        if col not in self._df.columns:
+            suggestion = suggest_column(col, list(self._df.columns))
+            raise ValueError(f"❌ Column '{col}' not found.\n{suggestion}")

dipencsv/engine/stream_engine.py

@@ -0,0 +1,91 @@
+# dipencsv/engine/stream_engine.py
+
+import pandas as pd
+from dipencsv.engine.base import BaseEngine
+from dipencsv.errors import suggest_column
+
+CHUNK_SIZE = 100_000
+
+
+class StreamEngine(BaseEngine):
+
+    def __init__(self, filepath: str, chunk_size: int = CHUNK_SIZE):
+        self._filepath = filepath
+        self._chunk_size = chunk_size
+        self._columns = self._get_columns()
+
+    def _get_columns(self) -> list:
+        df = pd.read_csv(self._filepath, nrows=1)
+        return list(df.columns)
+
+    def _chunks(self):
+        return pd.read_csv(self._filepath, chunksize=self._chunk_size)
+
+    def mean(self, col: str) -> float:
+        self._check_col(col)
+        total, count = 0, 0
+        for chunk in self._chunks():
+            total += chunk[col].sum()
+            count += chunk[col].count()
+        return total / count if count else 0
+
+    def median(self, col: str) -> float:
+        raise NotImplementedError("⚠️  median() not supported in stream mode — load a sample instead")
+
+    def max(self, col: str) -> float:
+        self._check_col(col)
+        result = None
+        for chunk in self._chunks():
+            chunk_max = chunk[col].max()
+            result = chunk_max if result is None else max(result, chunk_max)
+        return result
+
+    def min(self, col: str) -> float:
+        self._check_col(col)
+        result = None
+        for chunk in self._chunks():
+            chunk_min = chunk[col].min()
+            result = chunk_min if result is None else min(result, chunk_min)
+        return result
+
+    def count(self, col: str) -> int:
+        self._check_col(col)
+        total = 0
+        for chunk in self._chunks():
+            total += chunk[col].count()
+        return total
+
+    def group_mean(self, group_col: str, value_col: str) -> dict:
+        self._check_col(group_col)
+        self._check_col(value_col)
+        agg = {}
+        for chunk in self._chunks():
+            for key, grp in chunk.groupby(group_col):
+                if key not in agg:
+                    agg[key] = {"sum": 0, "count": 0}
+                agg[key]["sum"] += grp[value_col].sum()
+                agg[key]["count"] += grp[value_col].count()
+        return {k: v["sum"] / v["count"] for k, v in agg.items()}
+
+    def correlation(self, col1: str, col2: str) -> float:
+        raise NotImplementedError("⚠️  correlation() not supported in stream mode")
+
+    def filter(self, condition: str) -> pd.DataFrame:
+        results = []
+        for chunk in self._chunks():
+            filtered = chunk.query(condition)
+            results.append(filtered)
+        return pd.concat(results, ignore_index=True)
+
+    def top_n(self, col: str, n: int = 10) -> pd.DataFrame:
+        self._check_col(col)
+        result = pd.DataFrame()
+        for chunk in self._chunks():
+            result = pd.concat([result, chunk.nlargest(n, col)])
+            result = result.nlargest(n, col)
+        return result
+
+    def _check_col(self, col: str):
+        if col not in self._columns:
+            suggestion = suggest_column(col, self._columns)
+            raise ValueError(f"❌ Column '{col}' not found.\n{suggestion}")

dipencsv/errors.py

@@ -0,0 +1,23 @@
+# dipencsv/errors.py
+
+from difflib import get_close_matches
+
+
+def suggest_column(col: str, available_cols: list) -> str:
+    matches = get_close_matches(col, available_cols, n=3, cutoff=0.5)
+    if matches:
+        suggestions = "\n".join(f"  - {m}" for m in matches)
+        return f"💡 Did you mean:\n{suggestions}"
+    return f"💡 Available columns:\n" + "\n".join(f"  - {c}" for c in available_cols)
+
+
+def format_error(msg: str) -> str:
+    return f"❌ {msg}"
+
+
+def format_warning(msg: str) -> str:
+    return f"⚠️  {msg}"
+
+
+def format_success(msg: str) -> str:
+    return f"✅ {msg}"

dipencsv/exporter.py

@@ -0,0 +1,45 @@
+# dipencsv/exporter.py
+
+import json
+import pandas as pd
+from dipencsv.errors import format_success, format_error
+
+
+def export(df: pd.DataFrame, filepath: str):
+    if filepath.endswith(".csv"):
+        _export_csv(df, filepath)
+    elif filepath.endswith(".json"):
+        _export_json(df, filepath)
+    elif filepath.endswith(".xlsx"):
+        _export_excel(df, filepath)
+    else:
+        print(format_error(f"Unsupported format: {filepath}"))
+        print("💡 Supported: .csv, .json, .xlsx")
+
+
+def _export_csv(df: pd.DataFrame, filepath: str):
+    try:
+        df.to_csv(filepath, index=False)
+        print(format_success(f"Exported to {filepath}"))
+    except Exception as e:
+        print(format_error(f"Failed to export CSV: {e}"))
+
+
+def _export_json(df: pd.DataFrame, filepath: str):
+    try:
+        data = df.to_dict(orient="records")
+        with open(filepath, "w") as f:
+            json.dump(data, f, indent=2)
+        print(format_success(f"Exported to {filepath}"))
+    except Exception as e:
+        print(format_error(f"Failed to export JSON: {e}"))
+
+
+def _export_excel(df: pd.DataFrame, filepath: str):
+    try:
+        df.to_excel(filepath, index=False)
+        print(format_success(f"Exported to {filepath}"))
+    except ImportError:
+        print(format_error("openpyxl not installed — run: pip install openpyxl"))
+    except Exception as e:
+        print(format_error(f"Failed to export Excel: {e}"))

dipencsv/intelligence.py

@@ -0,0 +1,194 @@
+# dipencsv/intelligence.py
+
+import pandas as pd
+from dipencsv.errors import suggest_column
+
+
+def ask(df: pd.DataFrame, question: str):
+    q = question.lower().strip()
+
+    # group by — check FIRST before average/mean
+    if "by" in q:
+        result = _parse_group(q, df)
+        if result is not None:
+            print(result.to_string())
+            return result
+        else:
+            print("🤔 Could not parse group query.")
+            return None
+
+    # top N
+    if "top" in q:
+        n, col = _parse_top(q, df)
+        if col:
+            result = df.nlargest(n, col)
+            print(f"🏆 Top {n} by '{col}':")
+            print(result[[col]].to_string())
+            return result
+
+    # average / mean
+    if "average" in q or "mean" in q:
+        col = _find_col(q, df)
+        if col:
+            if not pd.api.types.is_numeric_dtype(df[col]):
+                print(f"❌ '{col}' is a text column — cannot calculate average")
+                return None
+            result = df[col].mean()
+            print(f"📊 Average {col}: {result:.2f}")
+            return result
+
+    # highest / maximum
+    if "highest" in q or "maximum" in q or "max" in q:
+        col = _find_col(q, df)
+        if col:
+            result = df[col].max()
+            print(f"📈 Highest {col}: {result}")
+            return result
+
+    # lowest / minimum
+    if "lowest" in q or "minimum" in q or "min" in q:
+        col = _find_col(q, df)
+        if col:
+            result = df[col].min()
+            print(f"📉 Lowest {col}: {result}")
+            return result
+
+    # count
+    if "count" in q or "how many" in q:
+        col = _find_col(q, df)
+        if col:
+            result = df[col].count()
+            print(f"🔢 Count of {col}: {result}")
+            return result
+
+    numeric_cols = list(df.select_dtypes(include="number").columns)
+    text_cols = list(df.select_dtypes(include=["object", "str"]).columns)
+
+    print("🤔 Could not understand the question.")
+    print(f"\n💡 Numeric columns (use for average/top/highest): {numeric_cols}")
+    print(f"💡 Text columns (use for grouping): {text_cols}")
+    print("\n💡 Try:")
+    if numeric_cols:
+        print(f"   - 'average {numeric_cols[0]}'")
+        print(f"   - 'top 10 {numeric_cols[0]}'")
+    if numeric_cols and text_cols:
+        print(f"   - 'highest {numeric_cols[0]} by {text_cols[0]}'")
+    return None
+
+
+def explain(df: pd.DataFrame):
+    print("=" * 40)
+    print("🧠 DATASET EXPLANATION")
+    print("=" * 40)
+
+    rows, cols = df.shape
+    print(f"\n📦 Dataset has {rows} rows and {cols} columns.")
+
+    numeric = df.select_dtypes(include="number")
+    text = df.select_dtypes(include=["object", "str"])
+
+    print(f"🔢 Numeric columns  : {list(numeric.columns)}")
+    print(f"🔤 Text columns     : {list(text.columns)}")
+
+    # trends
+    print("\n📈 TRENDS")
+    print("-" * 40)
+    for col in numeric.columns:
+        mean = numeric[col].mean()
+        std = numeric[col].std()
+        skew = numeric[col].skew()
+        skew_label = "right-skewed" if skew > 1 else "left-skewed" if skew < -1 else "normal"
+        print(f"  {col}: mean={mean:.2f}, std={std:.2f}, distribution={skew_label}")
+
+    # issues
+    print("\n⚠️  ISSUES")
+    print("-" * 40)
+    missing = df.isnull().sum()
+    missing = missing[missing > 0]
+    if missing.empty:
+        print("  ✅ No missing values")
+    else:
+        for col, count in missing.items():
+            pct = (count / rows) * 100
+            print(f"  ⚠️  '{col}' has {count} missing values ({pct:.1f}%)")
+
+    dupes = df.duplicated().sum()
+    if dupes > 0:
+        print(f"  ⚠️  {dupes} duplicate rows found")
+    else:
+        print("  ✅ No duplicates")
+
+    # suggestions
+    print("\n💡 SUGGESTIONS")
+    print("-" * 40)
+    if not missing.empty:
+        print("  - Run data.clean() to fix missing values")
+    if dupes > 0:
+        print("  - Run data.clean() to remove duplicates")
+    high_skew = [c for c in numeric.columns if abs(numeric[c].skew()) > 1]
+    if high_skew:
+        print(f"  - Columns with high skew: {high_skew} — consider log transform")
+    if not missing.empty or dupes == 0:
+        print("  - Data looks ready for analysis ✅")
+    print("=" * 40)
+
+
+def magic(df: pd.DataFrame):
+    print("=" * 40)
+    print("✨ MAGIC MODE")
+    print("=" * 40)
+
+    rows, cols = df.shape
+    missing = df.isnull().sum().sum()
+    dupes = df.duplicated().sum()
+    numeric = df.select_dtypes(include="number")
+
+    print(f"\n📦 {rows} rows × {cols} cols")
+    print(f"⚠️  Missing values : {missing}")
+    print(f"⚠️  Duplicates     : {dupes}")
+
+    print("\n📈 KEY INSIGHTS")
+    print("-" * 40)
+    for col in numeric.columns:
+        print(f"  {col}: min={numeric[col].min():.2f}, mean={numeric[col].mean():.2f}, max={numeric[col].max():.2f}")
+
+    print("\n🏆 RECOMMENDATION")
+    print("-" * 40)
+    if missing > 0 or dupes > 0:
+        print("  ⚠️  Run data.clean() before analysis")
+    else:
+        print("  ✅ Data is clean — ready to analyze")
+    print("=" * 40)
+
+
+# --- internal helpers ---
+
+def _find_col(question: str, df: pd.DataFrame):
+    words = question.lower().split()
+    for col in df.columns:
+        if col.lower() in words:  # match whole word not substring
+            return col
+    print("❌ No matching column found in your question.")
+    print(f"💡 Your columns: {list(df.columns)}")
+    return None
+
+def _parse_top(question: str, df: pd.DataFrame):
+    import re
+    n = 10
+    match = re.search(r"top\s+(\d+)", question)
+    if match:
+        n = int(match.group(1))
+    col = _find_col(question, df)
+    return n, col
+
+
+def _parse_group(question: str, df: pd.DataFrame):
+    words = question.lower().split()
+    cols = [c for c in df.columns if c.lower() in words]  # whole word match
+    numeric = df.select_dtypes(include="number")
+    if len(cols) >= 2:
+        group_col = cols[0]
+        value_col = cols[1]
+        if value_col in numeric.columns:
+            return df.groupby(group_col)[value_col].mean().round(2)
+    return None

dipencsv/loader.py

@@ -0,0 +1,24 @@
+# dipencsv/loader.py
+
+import os
+import pandas as pd
+
+
+def load_file(filepath: str) -> pd.DataFrame:
+    _validate(filepath)
+    df = pd.read_csv(filepath)
+    print(f"✅ Loaded: {df.shape[0]} rows × {df.shape[1]} cols")
+    return df
+
+
+def _validate(filepath: str):
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"❌ File not found: {filepath}")
+    if not filepath.endswith(".csv"):
+        raise ValueError(f"❌ Only CSV files supported: {filepath}")
+    if os.path.getsize(filepath) == 0:
+        raise ValueError(f"❌ File is empty: {filepath}")
+
+
+def get_file_size_mb(filepath: str) -> float:
+    return os.path.getsize(filepath) / (1024 * 1024)

dipencsv/summary.py

@@ -0,0 +1,103 @@
+# dipencsv/summary.py
+
+import pandas as pd
+from dipencsv.errors import format_warning
+
+
+def summary(df: pd.DataFrame):
+    print("=" * 40)
+    print("📊 DATASET SUMMARY")
+    print("=" * 40)
+    print(f"  Rows        : {df.shape[0]}")
+    print(f"  Columns     : {df.shape[1]}")
+    print(f"  Total cells : {df.shape[0] * df.shape[1]}")
+    print()
+
+    print("📋 COLUMN TYPES")
+    print("-" * 40)
+    for col, dtype in df.dtypes.items():
+        print(f"  {col:<25} {str(dtype)}")
+    print()
+
+    missing = df.isnull().sum()
+    missing = missing[missing > 0]
+    if not missing.empty:
+        print("⚠️  MISSING VALUES")
+        print("-" * 40)
+        for col, count in missing.items():
+            pct = (count / len(df)) * 100
+            print(f"  {col:<25} {count} missing ({pct:.1f}%)")
+        print()
+    else:
+        print("✅ No missing values found")
+        print()
+
+    print("🔢 NUMERIC COLUMNS")
+    print("-" * 40)
+    numeric = df.select_dtypes(include="number").columns.tolist()
+    if numeric:
+        for col in numeric:
+            print(f"  - {col}")
+    else:
+        print("  none")
+    print()
+
+    print("🔤 TEXT COLUMNS")
+    print("-" * 40)
+    text = df.select_dtypes(include=["object", "str"]).columns.tolist()
+    if text:
+        for col in text:
+            print(f"  - {col}")
+    else:
+        print("  none")
+    print("=" * 40)
+
+
+def report(df: pd.DataFrame):
+    print("=" * 40)
+    print("📝 FULL REPORT")
+    print("=" * 40)
+
+    # duplicates
+    dupes = df.duplicated().sum()
+    if dupes > 0:
+        print(format_warning(f"{dupes} duplicate rows found"))
+    else:
+        print("✅ No duplicates found")
+    print()
+
+    # missing %
+    print("📉 MISSING VALUES %")
+    print("-" * 40)
+    missing_pct = (df.isnull().sum() / len(df) * 100).round(2)
+    for col, pct in missing_pct.items():
+        status = "⚠️ " if pct > 0 else "✅"
+        print(f"  {status} {col:<23} {pct}%")
+    print()
+
+    # basic stats
+    print("📈 BASIC STATS")
+    print("-" * 40)
+    numeric_df = df.select_dtypes(include="number")
+    if not numeric_df.empty:
+        stats = numeric_df.describe().round(2)
+        print(stats.to_string())
+    else:
+        print("  No numeric columns")
+    print()
+
+    # warnings
+    print("⚠️  WARNINGS")
+    print("-" * 40)
+    warnings = []
+    if dupes > 0:
+        warnings.append(f"Duplicate rows: {dupes}")
+    high_missing = missing_pct[missing_pct > 30]
+    for col, pct in high_missing.items():
+        warnings.append(f"'{col}' has {pct}% missing values")
+    if warnings:
+        for w in warnings:
+            print(f"  ⚠️  {w}")
+    else:
+        print("  ✅ No major warnings")
+    print("=" * 40)

dipencsv-0.1.0.dist-info/METADATA

@@ -0,0 +1,239 @@
+Metadata-Version: 2.4
+Name: dipencsv
+Version: 0.1.0
+Summary: A beginner-friendly CSV analysis and ML data preparation toolkit
+Author: Dipendra
+License-Expression: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: openpyxl>=3.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+
+# DipenCSV 🐼
+
+> DipenCSV is to pandas what seaborn is to matplotlib — a friendlier, higher-level API.
+
+A beginner-friendly CSV analysis and ML data preparation toolkit built on top of pandas. Helps students and small teams clean, analyze, and prepare CSV data for machine learning — without needing to know pandas.
+
+## Who is it for?
+- 🎓 Students cleaning data for ML assignments
+- 🚀 Small startups needing quick data insights
+- 👨‍💻 Developers who work with CSVs occasionally
+- 📊 Anyone who finds pandas too complex
+
+## Installation
+
+```bash
+pip install dipencsv
+```
+
+## Quick Start
+
+```python
+from dipencsv import Data
+
+data = Data("your_file.csv")
+data.magic()  # one click everything
+```
+
+## Full ML Workflow in 15 lines
+
+```python
+from dipencsv import Data
+from sklearn.ensemble import RandomForestClassifier
+
+data = Data("titanic.csv")
+data.clean()
+data.drop(["passengerid", "name", "ticket", "cabin"])
+data.encode("sex")
+data.encode("embarked")
+data.remove_outliers("fare")
+data.normalize("fare")
+data.normalize("age")
+
+train, test = data.split(test_size=0.2)
+
+X_train = train.drop("survived", axis=1)
+y_train = train["survived"]
+X_test = test.drop("survived", axis=1)
+y_test = test["survived"]
+
+model = RandomForestClassifier()
+model.fit(X_train, y_train)
+print(f"Accuracy: {model.score(X_test, y_test):.2f}")
+# Accuracy: 0.76
+```
+
+---
+
+## All Commands
+
+### Loading
+```python
+data = Data("file.csv")                        # auto mode
+data = Data("bigfile.csv", stream=True)        # force stream mode
+data = Data("file.csv", auto_mode=False)       # force pandas mode
+```
+
+### Properties
+```python
+print(data.columns)   # list of column names
+print(data.shape)     # (rows, columns)
+```
+
+### Understanding Data
+```python
+data.summary()    # rows, cols, types, missing values
+data.report()     # duplicates, missing %, basic stats
+data.explain()    # trends, issues, suggestions
+data.magic()      # ⭐ one click full analysis
+```
+
+### Cleaning
+```python
+data.clean()                        # auto clean (safe mode)
+data.clean(strategy="aggressive")   # drop rows with missing values
+data.drop("column")                 # drop one column
+data.drop(["col1", "col2"])         # drop multiple columns
+data.rename("old_name", "new_name") # rename column
+data.fill("column", value)          # fill missing values manually
+```
+
+> ⚠️ After clean(), column names become lowercase with underscores.
+> Always check: `print(data.columns)`
+
+### ML Preparation
+```python
+data.encode("gender")           # text → numbers (label encoding)
+data.normalize("age")           # scale to [0, 1]
+data.standardize("salary")      # scale to mean=0, std=1
+data.remove_outliers("price")   # remove extreme values (IQR method)
+train, test = data.split(test_size=0.2)  # train/test split
+```
+
+### Analytics
+```python
+data.mean("salary")
+data.median("age")
+data.max("salary")
+data.min("salary")
+data.count("city")
+data.correlation("age", "salary")
+data.correlation_matrix()
+data.distribution("salary")
+data.outliers("salary")
+data.describe()
+data.value_counts("city")
+data.group_mean("city", "salary")
+```
+
+### Querying
+```python
+data.first()              # first 5 rows
+data.first(10)            # first 10 rows
+data.last()               # last 5 rows
+data.sort("age")          # sort ascending
+data.sort("age", asc=False)  # sort descending
+data.find("city", "Bangalore")  # find rows by value
+```
+
+### Intelligence
+```python
+data.ask("average salary")
+data.ask("top 10 salary")
+data.ask("highest salary by city")
+data.ask("lowest age")
+data.ask("count city")
+data.explain()
+data.magic()
+```
+
+### Export
+```python
+data.export("output.csv")    # CSV
+data.export("output.json")   # JSON
+data.export("output.xlsx")   # Excel (requires openpyxl)
+```
+
+---
+
+## Stream Mode (Big Files)
+
+DipenCSV auto detects file size and switches to stream mode for files > 500MB:
+
+```python
+data = Data("hugefile.csv")          # auto detects
+data = Data("hugefile.csv", stream=True)  # force stream
+```
+
+Supported in stream mode: `mean()`, `max()`, `min()`, `count()`, `group_mean()`, `filter()`, `top_n()`
+
+---
+
+## Smart Error Handling
+
+```python
+data.mean("salry")
+# ❌ Column 'salry' not found.
+# 💡 Did you mean:
+#   - salary
+#   - salary_usd
+```
+
+---
+
+## Common Issues
+
+**KeyError after clean()**
+```python
+data.clean()
+print(data.columns)      # check actual column names
+data.mean("salary")      # use lowercase
+```
+
+**Excel export failing**
+```bash
+pip install openpyxl
+```
+
+**Big file crashes**
+```python
+data = Data("bigfile.csv", stream=True)
+```
+
+---
+
+## Dependencies
+- `pandas >= 1.5.0`
+- `scikit-learn >= 1.0.0`
+- `openpyxl >= 3.0.0` (optional, Excel export)
+
+## Documentation
+Full docs available in the `docs/` folder.
+## Documentation
+
+| Doc | What it covers |
+|-----|----------------|
+| [Getting Started](docs/getting_started.md) | installation, quick start |
+| [Loading Data](docs/loading_data.md) | Data(), auto mode, stream mode |
+| [Properties](docs/properties.md) | columns, shape |
+| [Understanding Data](docs/understanding_data.md) | summary, report, explain, magic |
+| [Cleaning Data](docs/cleaning_data.md) | clean, drop, rename, fill |
+| [ML Preparation](docs/ml_prep.md) | encode, normalize, split etc |
+| [Analytics](docs/analytics.md) | mean, correlation, outliers etc |
+| [Query](docs/query.md) | sort, find, first, last |
+| [Intelligence](docs/intelligence.md) | ask, explain, magic |
+| [Stream Mode](docs/stream_mode.md) | big file handling |
+| [Export](docs/export.md) | csv, json, xlsx |
+| [Examples](docs/examples.md) | real world examples |
+| [FAQ](docs/faq.md) | common student questions |
+| [ML Workflow](docs/ml_workflow.md) | full end-to-end ML example |
+
+## License
+MIT License
+
+## Author
+Built by Dipendra — a CS student who wanted pandas to be less painful.

dipencsv-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+dipencsv/__init__.py,sha256=tt4CaUaGC7ReFCulu9Aiot29paRWhQWzDali3B3O7JE,121
+dipencsv/analytics.py,sha256=nqni-CL9ekMaEOvSLk1VAcYiShMfsi4XMUFvhWA6EZc,2257
+dipencsv/cleaner.py,sha256=jVyyegwGWyK0UUPYmrKDjebmci6hYvq7t5DmuT5htrI,4860
+dipencsv/core.py,sha256=Z2GGR7s7qoni5zLXB8o5TYRqnI0wLX9IYuv5Ot3lOs0,8050
+dipencsv/errors.py,sha256=NMLDt01VSr4KZMLdH5qeuJkrUwvYLOopjWGDG5Ebu8U,598
+dipencsv/exporter.py,sha256=pUhjidm4F1r6VYOYNDxufeRfI-bNdKUuWqhN-X_PDPY,1428
+dipencsv/intelligence.py,sha256=NYQFwT68sLLj4oECcknZLC4fYgBcBEwXG6Mq4p8t5PY,6089
+dipencsv/loader.py,sha256=7UnG2Hzde08SN0FN-22pRrruiZEUHqx9Tyy-AuwnUWI,678
+dipencsv/summary.py,sha256=_qwYZsytcO7SzGXorYtK5T1FO-Kbes9TeAGdjSi0Y8E,2711
+dipencsv/engine/__init__.py,sha256=6f-E6-atDCyRVVgDAg4--OJal5aBBs2xKAD4y46xUu4,184
+dipencsv/engine/base.py,sha256=D8CFeRUd_d7HKrIF48sgCfN3_rGny3ebrvSsxsJIXdU,788
+dipencsv/engine/pandas_engine.py,sha256=hzilZ-12C_5nTTiz2lMNn3EkObnzBl4gyCnpKzrX1gk,1594
+dipencsv/engine/stream_engine.py,sha256=IhCx42n5fZ59fe6RGaG-8-GsXi20-odX698PahSsAhM,3138
+dipencsv-0.1.0.dist-info/METADATA,sha256=sM7xwAtwHkh38RQzQTIw5NY2Gp5tASgzDbcy7VRL_Zo,6181
+dipencsv-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+dipencsv-0.1.0.dist-info/top_level.txt,sha256=vT6kHGNkvpiVjSve9w6c8fovS-RX9qun6DprSl3bSeI,9
+dipencsv-0.1.0.dist-info/RECORD,,

dipencsv-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dipencsv-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dipencsv