murthylytics 0.1.0__py3-none-any.whl

murthylytics/__init__.py

@@ -0,0 +1,274 @@
+"""Murthylytics — production-grade, AI-powered ML data analysis & preparation.
+
+The package exposes a **stateful facade**: load a dataset once and every other
+call operates on the active session::
+
+    import murthylytics as mlt
+
+    mlt.read_csv("data.csv")
+    mlt.summary()
+    mlt.inspect()
+
+Advanced users can instead work with explicit :class:`~murthylytics.core.Session`
+objects and the pure engine functions in the sub-packages.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+from . import exploration as _exploration
+from . import io as _io
+from ._version import __version__
+from .core import (
+    Config,
+    Session,
+    enable_logging,
+    get_session,
+    reset_session,
+)
+from .core.context import get_session as _get_session
+from .core.exceptions import MurthylyticsError, NoActiveDatasetError
+
+__all__ = [
+    "__version__",
+    # data I/O
+    "read_csv",
+    "read_excel",
+    "read_json",
+    "read_parquet",
+    "read_sql",
+    # exploration
+    "show",
+    "top",
+    "bottom",
+    "shape",
+    "columns",
+    "summary",
+    "describe",
+    "info",
+    "types",
+    "missing",
+    "duplicates",
+    "memory",
+    "inspect",
+    # workflow (implemented incrementally)
+    "clean_data",
+    "cleaned_pyplot",
+    "auto_eda",
+    "preprocess",
+    "engineer_features",
+    "select_features",
+    "recommend_model",
+    "export_pipeline",
+    "use_plugin",
+    # session management
+    "set_target",
+    "reset",
+    "get_dataframe",
+    "get_session",
+    "reset_session",
+    "enable_logging",
+    "Config",
+    "Session",
+    "MurthylyticsError",
+]
+
+
+def _df() -> pd.DataFrame:
+    """Return the active session's dataframe or raise if none is loaded."""
+    return _get_session().df
+
+
+# ---------------------------------------------------------------------------
+# Data I/O — read, then install into the active session.
+# ---------------------------------------------------------------------------
+def read_csv(path: Any, **kwargs: Any) -> pd.DataFrame:
+    """Read a CSV file and make it the active dataset. See :func:`io.read_csv`."""
+    session = _get_session()
+    df = _io.read_csv(path, config=session.config, **kwargs)
+    return session.load(df, source=str(path))
+
+
+def read_excel(path: Any, **kwargs: Any) -> pd.DataFrame:
+    """Read an Excel file and make it the active dataset."""
+    session = _get_session()
+    df = _io.read_excel(path, config=session.config, **kwargs)
+    return session.load(df, source=str(path))
+
+
+def read_json(path: Any, **kwargs: Any) -> pd.DataFrame:
+    """Read a JSON file and make it the active dataset."""
+    session = _get_session()
+    df = _io.read_json(path, config=session.config, **kwargs)
+    return session.load(df, source=str(path))
+
+
+def read_parquet(path: Any, **kwargs: Any) -> pd.DataFrame:
+    """Read a Parquet file and make it the active dataset."""
+    session = _get_session()
+    df = _io.read_parquet(path, config=session.config, **kwargs)
+    return session.load(df, source=str(path))
+
+
+def read_sql(query: str, con: Any, **kwargs: Any) -> pd.DataFrame:
+    """Run a SQL query and make the result the active dataset."""
+    session = _get_session()
+    df = _io.read_sql(query, con, config=session.config, **kwargs)
+    return session.load(df, source="sql")
+
+
+# ---------------------------------------------------------------------------
+# Exploration.
+# ---------------------------------------------------------------------------
+def show(n: Optional[int] = None) -> pd.DataFrame:
+    """Return the active dataframe (or its first ``n`` rows)."""
+    df = _df()
+    return df if n is None else df.head(n)
+
+
+def top(n: int = 5) -> pd.DataFrame:
+    """Return the first ``n`` rows of the active dataset."""
+    return _exploration.top(_df(), n)
+
+
+def bottom(n: int = 5) -> pd.DataFrame:
+    """Return the last ``n`` rows of the active dataset."""
+    return _exploration.bottom(_df(), n)
+
+
+def shape() -> Dict[str, int]:
+    """Return row/column counts of the active dataset."""
+    return _exploration.shape(_df())
+
+
+def columns() -> List[str]:
+    """Return the active dataset's column names."""
+    return _exploration.columns(_df())
+
+
+def summary() -> Dict[str, Any]:
+    """Return a compact overview of the active dataset."""
+    return _exploration.summary(_df())
+
+
+def describe(include: str = "all") -> pd.DataFrame:
+    """Return descriptive statistics for the active dataset."""
+    return _exploration.describe(_df(), include=include)
+
+
+def info() -> Dict[str, Any]:
+    """Return structured dtype/non-null/memory info for the active dataset."""
+    return _exploration.info(_df())
+
+
+def types() -> Dict[str, str]:
+    """Return a column -> dtype mapping for the active dataset."""
+    return _exploration.types(_df())
+
+
+def missing() -> pd.DataFrame:
+    """Return per-column missing-value counts and percentages."""
+    return _exploration.missing(_df())
+
+
+def duplicates() -> Dict[str, Any]:
+    """Return duplicate-row statistics for the active dataset."""
+    return _exploration.duplicates(_df())
+
+
+def memory() -> Dict[str, float]:
+    """Return a per-dtype memory footprint report (MB)."""
+    return _exploration.memory(_df())
+
+
+def inspect() -> "_exploration.InspectionReport":
+    """Run a full intelligent inspection of the active dataset."""
+    session = _get_session()
+    report = _exploration.inspect(session.df, config=session.config)
+    if report.potential_targets and session.target is None:
+        session.metadata["suggested_target"] = report.potential_targets[0]
+    return report
+
+
+# ---------------------------------------------------------------------------
+# Session management.
+# ---------------------------------------------------------------------------
+def set_target(column: str) -> None:
+    """Declare the target/label column for downstream steps."""
+    session = _get_session()
+    if column not in session.df.columns:
+        raise MurthylyticsError(f"Column '{column}' is not in the active dataset.")
+    session.target = column
+    session.record("set_target", column=column)
+
+
+def reset() -> pd.DataFrame:
+    """Restore the active dataframe to its originally loaded state."""
+    session = _get_session()
+    session.reset()
+    return session.df
+
+
+def get_dataframe() -> pd.DataFrame:
+    """Return the active dataframe (a live reference, not a copy)."""
+    return _df()
+
+
+# ---------------------------------------------------------------------------
+# Workflow API — implemented in later iterations. These raise a clear,
+# actionable error today so the public surface is stable and discoverable.
+# ---------------------------------------------------------------------------
+def _not_yet(name: str, iteration: str) -> "NotImplementedError":
+    return NotImplementedError(
+        f"`mlt.{name}()` is part of the Murthylytics roadmap ({iteration}). "
+        f"The function is reserved and its signature is stable; the engine lands "
+        f"in an upcoming release."
+    )
+
+
+def clean_data(**kwargs: Any) -> pd.DataFrame:
+    """Automatically clean the active dataset (roadmap: cleaning module)."""
+    raise _not_yet("clean_data", "iteration 2 — cleaning")
+
+
+def cleaned_pyplot(**kwargs: Any) -> Any:
+    """Auto-generate diagnostic plots (roadmap: viz module)."""
+    raise _not_yet("cleaned_pyplot", "iteration 3 — visualization")
+
+
+def auto_eda(**kwargs: Any) -> Any:
+    """Generate HTML/MD/JSON/PDF EDA reports (roadmap: eda module)."""
+    raise _not_yet("auto_eda", "iteration 4 — AutoEDA reporting")
+
+
+def preprocess(**kwargs: Any) -> Any:
+    """Detect the task and build a preprocessing pipeline (roadmap: preprocess)."""
+    raise _not_yet("preprocess", "iteration 5 — preprocessing")
+
+
+def engineer_features(**kwargs: Any) -> Any:
+    """Automatically engineer features (roadmap: features module)."""
+    raise _not_yet("engineer_features", "iteration 6 — feature engineering")
+
+
+def select_features(**kwargs: Any) -> Any:
+    """Select the most informative features (roadmap: features module)."""
+    raise _not_yet("select_features", "iteration 6 — feature selection")
+
+
+def recommend_model(**kwargs: Any) -> Any:
+    """Recommend suitable models with justification (roadmap: modeling)."""
+    raise _not_yet("recommend_model", "iteration 7 — model recommendation")
+
+
+def export_pipeline(path: Any, **kwargs: Any) -> Any:
+    """Export the reproducible preprocessing pipeline (roadmap: pipeline)."""
+    raise _not_yet("export_pipeline", "iteration 8 — pipeline export")
+
+
+def use_plugin(name: str, **kwargs: Any) -> Any:
+    """Activate a domain plugin, e.g. ``mlt.use_plugin('malware')`` (roadmap)."""
+    raise _not_yet("use_plugin", "iteration 9 — plugin architecture")

murthylytics/_version.py

@@ -0,0 +1,3 @@
+"""Single source of truth for the package version (read by hatchling)."""
+
+__version__ = "0.1.0"

murthylytics/cleaning/__init__.py

@@ -0,0 +1 @@
+"""Murthylytics cleaning module (roadmap — see project README for status)."""

murthylytics/core/__init__.py

@@ -0,0 +1,48 @@
+"""Core infrastructure: config, session/context, logging, types, exceptions."""
+
+from __future__ import annotations
+
+from .config import DEFAULT_CONFIG, Config
+from .context import (
+    Operation,
+    Session,
+    get_session,
+    reset_session,
+    set_session,
+)
+from .exceptions import (
+    ConfigurationError,
+    DataLoadError,
+    DetectionError,
+    MissingDependencyError,
+    MurthylyticsError,
+    NoActiveDatasetError,
+    NotFittedError,
+    PluginError,
+)
+from .logging import disable_logging, enable_logging, get_logger
+from .types import ColumnRole, Severity, TaskType
+
+__all__ = [
+    "Config",
+    "DEFAULT_CONFIG",
+    "Session",
+    "Operation",
+    "get_session",
+    "set_session",
+    "reset_session",
+    "MurthylyticsError",
+    "NoActiveDatasetError",
+    "DataLoadError",
+    "DetectionError",
+    "MissingDependencyError",
+    "ConfigurationError",
+    "PluginError",
+    "NotFittedError",
+    "get_logger",
+    "enable_logging",
+    "disable_logging",
+    "ColumnRole",
+    "TaskType",
+    "Severity",
+]

murthylytics/core/config.py

@@ -0,0 +1,88 @@
+"""User-tunable configuration for Murthylytics.
+
+A single immutable-by-convention :class:`Config` dataclass carries the knobs that
+influence automatic detection, cleaning heuristics and inspection thresholds.
+Defaults are chosen to be sensible for medium-sized tabular datasets.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Any, Dict
+
+from .exceptions import ConfigurationError
+
+
+@dataclass(frozen=True)
+class Config:
+    """Global configuration knobs.
+
+    Attributes
+    ----------
+    sample_rows:
+        Number of rows sampled for expensive detection routines (encoding,
+        delimiter, dtype inference) on large files.
+    high_cardinality_ratio:
+        A categorical column whose unique-count / row-count exceeds this ratio
+        is flagged as high cardinality.
+    id_unique_ratio:
+        Columns whose unique-value ratio is at least this are considered
+        ID-like candidates for removal.
+    constant_threshold:
+        Columns where the most frequent value covers at least this fraction of
+        rows are treated as (quasi-)constant.
+    missing_drop_column_ratio:
+        Columns missing at least this fraction of values are proposed for removal.
+    outlier_zscore_threshold:
+        Absolute z-score above which a point is an outlier under the z-score rule.
+    outlier_iqr_multiplier:
+        IQR multiplier (Tukey's fences) for the IQR outlier rule.
+    correlation_threshold:
+        Absolute correlation above which a numeric pair is reported as strongly
+        correlated (and a data-leakage candidate against the target).
+    class_imbalance_ratio:
+        minority/majority class-frequency ratio below which a target is flagged
+        as imbalanced.
+    memory_optimize:
+        Whether readers downcast numeric/categorical dtypes to save memory.
+    random_state:
+        Seed used by any stochastic routine for reproducibility.
+    """
+
+    sample_rows: int = 10_000
+    high_cardinality_ratio: float = 0.5
+    id_unique_ratio: float = 0.95
+    constant_threshold: float = 0.99
+    missing_drop_column_ratio: float = 0.6
+    outlier_zscore_threshold: float = 3.0
+    outlier_iqr_multiplier: float = 1.5
+    correlation_threshold: float = 0.9
+    class_imbalance_ratio: float = 0.2
+    memory_optimize: bool = True
+    random_state: int = 42
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.sample_rows <= 0:
+            raise ConfigurationError("sample_rows must be a positive integer.")
+        for name in (
+            "high_cardinality_ratio",
+            "id_unique_ratio",
+            "constant_threshold",
+            "missing_drop_column_ratio",
+            "class_imbalance_ratio",
+        ):
+            value = getattr(self, name)
+            if not 0.0 <= value <= 1.0:
+                raise ConfigurationError(f"{name} must be within [0, 1]; got {value}.")
+
+    def with_overrides(self, **overrides: Any) -> "Config":
+        """Return a new :class:`Config` with the given fields replaced."""
+        valid = {f for f in self.__dataclass_fields__ if f != "extra"}
+        known = {k: v for k, v in overrides.items() if k in valid}
+        unknown = {k: v for k, v in overrides.items() if k not in valid}
+        new_extra = {**self.extra, **unknown}
+        return replace(self, extra=new_extra, **known)
+
+
+DEFAULT_CONFIG = Config()

murthylytics/core/context.py

@@ -0,0 +1,125 @@
+"""The stateful session that backs the ``import murthylytics as mlt`` facade.
+
+The facade functions (``mlt.read_csv``, ``mlt.summary`` ...) are thin wrappers
+that read from / write to a single active :class:`Session`. Keeping *all* mutable
+state in one object means the underlying engine modules stay pure and testable,
+and it lets advanced users work with explicit :class:`Session` objects instead
+of the global one when they need isolation (e.g. threads, notebooks, tests).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from .config import DEFAULT_CONFIG, Config
+from .exceptions import NoActiveDatasetError
+
+if TYPE_CHECKING:  # pragma: no cover
+    import pandas as pd
+
+
+@dataclass
+class Operation:
+    """A single recorded step in the session's provenance log."""
+
+    name: str
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+class Session:
+    """Holds the active dataframe and everything derived from it.
+
+    Parameters
+    ----------
+    config:
+        Configuration knobs; falls back to :data:`DEFAULT_CONFIG`.
+    """
+
+    def __init__(self, config: Optional[Config] = None) -> None:
+        self.config: Config = config or DEFAULT_CONFIG
+        self._df: Optional["pd.DataFrame"] = None
+        self._raw_df: Optional["pd.DataFrame"] = None  # untouched original
+        self.source: Optional[str] = None
+        self.target: Optional[str] = None
+        self.metadata: Dict[str, Any] = {}
+        self.history: List[Operation] = []
+
+    # -- dataframe access -------------------------------------------------
+    @property
+    def df(self) -> "pd.DataFrame":
+        """The active (possibly transformed) dataframe.
+
+        Raises
+        ------
+        NoActiveDatasetError
+            If no dataset has been loaded.
+        """
+        if self._df is None:
+            raise NoActiveDatasetError()
+        return self._df
+
+    @df.setter
+    def df(self, value: "pd.DataFrame") -> None:
+        self._df = value
+
+    @property
+    def has_data(self) -> bool:
+        return self._df is not None
+
+    @property
+    def raw_df(self) -> "pd.DataFrame":
+        """The original dataframe as first loaded, before any transforms."""
+        if self._raw_df is None:
+            raise NoActiveDatasetError()
+        return self._raw_df
+
+    def load(
+        self,
+        df: "pd.DataFrame",
+        *,
+        source: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> "pd.DataFrame":
+        """Install ``df`` as the active dataset and reset derived state."""
+        self._df = df
+        self._raw_df = df.copy(deep=True)
+        self.source = source
+        self.target = None
+        self.metadata = metadata or {}
+        self.history = [Operation("load", {"source": source, "shape": tuple(df.shape)})]
+        return df
+
+    def record(self, name: str, **details: Any) -> None:
+        """Append an entry to the provenance log."""
+        self.history.append(Operation(name, details))
+
+    def reset(self) -> None:
+        """Restore the active dataframe to its originally loaded state."""
+        if self._raw_df is None:
+            raise NoActiveDatasetError()
+        self._df = self._raw_df.copy(deep=True)
+        self.record("reset")
+
+
+# --------------------------------------------------------------------------
+# Global singleton used by the facade.
+# --------------------------------------------------------------------------
+_active_session: Session = Session()
+
+
+def get_session() -> Session:
+    """Return the process-wide active :class:`Session`."""
+    return _active_session
+
+
+def set_session(session: Session) -> Session:
+    """Replace the active session (mainly for advanced users and tests)."""
+    global _active_session
+    _active_session = session
+    return _active_session
+
+
+def reset_session(config: Optional[Config] = None) -> Session:
+    """Discard all state and start a fresh active session."""
+    return set_session(Session(config=config))

murthylytics/core/exceptions.py

@@ -0,0 +1,54 @@
+"""Exception hierarchy for Murthylytics.
+
+All library-specific errors derive from :class:`MurthylyticsError`, so callers
+can catch everything the library raises with a single ``except``.
+"""
+
+from __future__ import annotations
+
+
+class MurthylyticsError(Exception):
+    """Base class for every error raised by Murthylytics."""
+
+
+class NoActiveDatasetError(MurthylyticsError):
+    """Raised when a facade call needs data but none has been loaded yet."""
+
+    def __init__(self, message: str | None = None) -> None:
+        super().__init__(
+            message
+            or "No active dataset. Load one first, e.g. `mlt.read_csv('data.csv')`."
+        )
+
+
+class DataLoadError(MurthylyticsError):
+    """Raised when a dataset cannot be read or parsed."""
+
+
+class DetectionError(MurthylyticsError):
+    """Raised when automatic detection (encoding, delimiter, dtype) fails."""
+
+
+class MissingDependencyError(MurthylyticsError):
+    """Raised when an optional dependency is required but not installed."""
+
+    def __init__(self, package: str, extra: str | None = None) -> None:
+        hint = f"pip install 'murthylytics[{extra}]'" if extra else f"pip install {package}"
+        super().__init__(
+            f"The optional dependency '{package}' is required for this feature. "
+            f"Install it with: {hint}"
+        )
+        self.package = package
+        self.extra = extra
+
+
+class ConfigurationError(MurthylyticsError):
+    """Raised when configuration values are invalid or inconsistent."""
+
+
+class PluginError(MurthylyticsError):
+    """Raised for plugin registration or execution failures."""
+
+
+class NotFittedError(MurthylyticsError):
+    """Raised when a pipeline/transform is used before being fitted."""

murthylytics/core/logging.py

@@ -0,0 +1,75 @@
+"""Centralised logging configuration for Murthylytics.
+
+Uses a single library-root logger (``murthylytics``). Following best practice
+for libraries, we attach a :class:`~logging.NullHandler` by default so importing
+the package never emits output; :func:`enable_logging` opts the user in.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+_ROOT_NAME = "murthylytics"
+
+# Attach a NullHandler once so "No handler found" warnings never appear.
+logging.getLogger(_ROOT_NAME).addHandler(logging.NullHandler())
+
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+    """Return a child logger under the ``murthylytics`` namespace.
+
+    Parameters
+    ----------
+    name:
+        Dotted suffix, typically ``__name__`` of the calling module. When it
+        already lives under the package, it is returned unchanged.
+    """
+    if not name or name == _ROOT_NAME:
+        return logging.getLogger(_ROOT_NAME)
+    if name.startswith(_ROOT_NAME + "."):
+        return logging.getLogger(name)
+    # Turn "murthylytics.io.readers" style module paths into a clean suffix.
+    suffix = name.split(".")[-1]
+    return logging.getLogger(f"{_ROOT_NAME}.{suffix}")
+
+
+def enable_logging(level: int | str = logging.INFO) -> None:
+    """Turn on console logging for the library.
+
+    Parameters
+    ----------
+    level:
+        A :mod:`logging` level (``logging.DEBUG``) or its name (``"DEBUG"``).
+    """
+    logger = logging.getLogger(_ROOT_NAME)
+    if isinstance(level, str):
+        level = logging.getLevelName(level.upper())
+    logger.setLevel(level)
+
+    # Avoid stacking duplicate stream handlers on repeated calls.
+    for handler in logger.handlers:
+        if isinstance(handler, logging.StreamHandler) and not isinstance(
+            handler, logging.NullHandler
+        ):
+            handler.setLevel(level)
+            return
+
+    handler = logging.StreamHandler()
+    handler.setLevel(level)
+    handler.setFormatter(
+        logging.Formatter(
+            "%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
+            datefmt="%H:%M:%S",
+        )
+    )
+    logger.addHandler(handler)
+
+
+def disable_logging() -> None:
+    """Silence the library logger again."""
+    logger = logging.getLogger(_ROOT_NAME)
+    for handler in list(logger.handlers):
+        if not isinstance(handler, logging.NullHandler):
+            logger.removeHandler(handler)
+    logger.setLevel(logging.WARNING)