murthylytics 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- murthylytics/__init__.py +274 -0
- murthylytics/_version.py +3 -0
- murthylytics/cleaning/__init__.py +1 -0
- murthylytics/core/__init__.py +48 -0
- murthylytics/core/config.py +88 -0
- murthylytics/core/context.py +125 -0
- murthylytics/core/exceptions.py +54 -0
- murthylytics/core/logging.py +75 -0
- murthylytics/core/optional.py +32 -0
- murthylytics/core/types.py +50 -0
- murthylytics/eda/__init__.py +1 -0
- murthylytics/exploration/__init__.py +36 -0
- murthylytics/exploration/display.py +99 -0
- murthylytics/exploration/inspect.py +337 -0
- murthylytics/features/__init__.py +1 -0
- murthylytics/io/__init__.py +28 -0
- murthylytics/io/detect.py +125 -0
- murthylytics/io/optimize.py +84 -0
- murthylytics/io/readers.py +160 -0
- murthylytics/modeling/__init__.py +1 -0
- murthylytics/pipeline/__init__.py +1 -0
- murthylytics/plugins/__init__.py +1 -0
- murthylytics/preprocess/__init__.py +1 -0
- murthylytics/py.typed +0 -0
- murthylytics/viz/__init__.py +1 -0
- murthylytics-0.1.0.dist-info/METADATA +168 -0
- murthylytics-0.1.0.dist-info/RECORD +29 -0
- murthylytics-0.1.0.dist-info/WHEEL +4 -0
- murthylytics-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Automatic detection of encoding, delimiter and datetime columns.
|
|
2
|
+
|
|
3
|
+
These routines are deliberately dependency-light: they fall back to the standard
|
|
4
|
+
library (``csv.Sniffer``, byte heuristics) when optional packages such as
|
|
5
|
+
``charset-normalizer`` are unavailable, so basic CSV reading works out of the box.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import List, Optional
|
|
13
|
+
|
|
14
|
+
import pandas as pd
|
|
15
|
+
|
|
16
|
+
from ..core.logging import get_logger
|
|
17
|
+
from ..core.optional import has_module
|
|
18
|
+
|
|
19
|
+
logger = get_logger(__name__)
|
|
20
|
+
|
|
21
|
+
_COMMON_DELIMITERS = [",", ";", "\t", "|", ":"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def detect_encoding(path: str, *, sample_bytes: int = 65_536) -> str:
|
|
25
|
+
"""Best-effort text-encoding detection for a file.
|
|
26
|
+
|
|
27
|
+
Uses ``charset-normalizer`` when installed, otherwise a BOM + UTF-8 probe,
|
|
28
|
+
falling back to ``latin-1`` (which can decode any byte stream).
|
|
29
|
+
"""
|
|
30
|
+
raw = Path(path).read_bytes()[:sample_bytes]
|
|
31
|
+
|
|
32
|
+
# Byte-order marks are unambiguous.
|
|
33
|
+
if raw.startswith(b"\xef\xbb\xbf"):
|
|
34
|
+
return "utf-8-sig"
|
|
35
|
+
if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
36
|
+
return "utf-16"
|
|
37
|
+
|
|
38
|
+
if has_module("charset_normalizer"):
|
|
39
|
+
from charset_normalizer import from_bytes # local import
|
|
40
|
+
|
|
41
|
+
result = from_bytes(raw).best()
|
|
42
|
+
if result is not None and result.encoding:
|
|
43
|
+
logger.debug("charset-normalizer detected encoding=%s", result.encoding)
|
|
44
|
+
return result.encoding
|
|
45
|
+
|
|
46
|
+
# Standard-library fallback: try strict UTF-8, else latin-1.
|
|
47
|
+
try:
|
|
48
|
+
raw.decode("utf-8")
|
|
49
|
+
return "utf-8"
|
|
50
|
+
except UnicodeDecodeError:
|
|
51
|
+
logger.debug("Falling back to latin-1 encoding for %s", path)
|
|
52
|
+
return "latin-1"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def detect_delimiter(
|
|
56
|
+
path: str,
|
|
57
|
+
*,
|
|
58
|
+
encoding: str = "utf-8",
|
|
59
|
+
sample_bytes: int = 32_768,
|
|
60
|
+
) -> str:
|
|
61
|
+
"""Detect the field delimiter of a delimited text file.
|
|
62
|
+
|
|
63
|
+
Tries :class:`csv.Sniffer` first, then a frequency heuristic over common
|
|
64
|
+
delimiters. Defaults to ``,`` if nothing conclusive is found.
|
|
65
|
+
"""
|
|
66
|
+
with open(path, encoding=encoding, errors="replace") as fh:
|
|
67
|
+
sample = fh.read(sample_bytes)
|
|
68
|
+
|
|
69
|
+
if not sample:
|
|
70
|
+
return ","
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
dialect = csv.Sniffer().sniff(sample, delimiters="".join(_COMMON_DELIMITERS))
|
|
74
|
+
logger.debug("csv.Sniffer detected delimiter=%r", dialect.delimiter)
|
|
75
|
+
return dialect.delimiter
|
|
76
|
+
except csv.Error:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
# Frequency heuristic on the first non-empty line.
|
|
80
|
+
first_line = next((ln for ln in sample.splitlines() if ln.strip()), "")
|
|
81
|
+
counts = {d: first_line.count(d) for d in _COMMON_DELIMITERS}
|
|
82
|
+
best = max(counts, key=counts.get)
|
|
83
|
+
if counts[best] == 0:
|
|
84
|
+
return ","
|
|
85
|
+
logger.debug("Heuristic detected delimiter=%r (counts=%s)", best, counts)
|
|
86
|
+
return best
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def detect_datetime_columns(
|
|
90
|
+
df: pd.DataFrame,
|
|
91
|
+
*,
|
|
92
|
+
sample_rows: int = 1_000,
|
|
93
|
+
success_ratio: float = 0.9,
|
|
94
|
+
) -> List[str]:
|
|
95
|
+
"""Return object columns that parse cleanly as datetimes.
|
|
96
|
+
|
|
97
|
+
A column is considered datetime-like when at least ``success_ratio`` of its
|
|
98
|
+
non-null sampled values parse without error.
|
|
99
|
+
"""
|
|
100
|
+
candidates: List[str] = []
|
|
101
|
+
obj_cols = df.select_dtypes(include=["object", "string"]).columns
|
|
102
|
+
sample = df.head(sample_rows)
|
|
103
|
+
|
|
104
|
+
for col in obj_cols:
|
|
105
|
+
non_null = sample[col].dropna()
|
|
106
|
+
if non_null.empty:
|
|
107
|
+
continue
|
|
108
|
+
parsed = pd.to_datetime(non_null, errors="coerce", format="mixed")
|
|
109
|
+
ratio = parsed.notna().mean()
|
|
110
|
+
if ratio >= success_ratio:
|
|
111
|
+
candidates.append(col)
|
|
112
|
+
logger.debug("Column %r looks datetime-like (%.0f%%)", col, ratio * 100)
|
|
113
|
+
return candidates
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def coerce_datetimes(df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:
|
|
117
|
+
"""Convert the given (or auto-detected) columns to datetime dtype in place-ish.
|
|
118
|
+
|
|
119
|
+
Returns the same dataframe with the columns converted; parsing errors become
|
|
120
|
+
``NaT`` so this never raises on messy data.
|
|
121
|
+
"""
|
|
122
|
+
cols = columns if columns is not None else detect_datetime_columns(df)
|
|
123
|
+
for col in cols:
|
|
124
|
+
df[col] = pd.to_datetime(df[col], errors="coerce", format="mixed")
|
|
125
|
+
return df
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Memory optimisation for pandas dataframes via dtype downcasting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Dict
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from ..core.logging import get_logger
|
|
10
|
+
|
|
11
|
+
logger = get_logger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def memory_usage_bytes(df: pd.DataFrame) -> int:
|
|
15
|
+
"""Total deep memory usage of ``df`` in bytes."""
|
|
16
|
+
return int(df.memory_usage(deep=True).sum())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def optimize_memory(
|
|
20
|
+
df: pd.DataFrame,
|
|
21
|
+
*,
|
|
22
|
+
categorical_ratio: float = 0.5,
|
|
23
|
+
copy: bool = True,
|
|
24
|
+
) -> pd.DataFrame:
|
|
25
|
+
"""Downcast numeric columns and convert low-cardinality objects to category.
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
categorical_ratio:
|
|
30
|
+
Object columns whose unique/row ratio is below this become ``category``.
|
|
31
|
+
copy:
|
|
32
|
+
When True (default) the input is left untouched and a new frame returned.
|
|
33
|
+
|
|
34
|
+
Returns
|
|
35
|
+
-------
|
|
36
|
+
pandas.DataFrame
|
|
37
|
+
A dtype-optimised dataframe. Values are preserved; only storage changes.
|
|
38
|
+
"""
|
|
39
|
+
out = df.copy() if copy else df
|
|
40
|
+
before = memory_usage_bytes(out)
|
|
41
|
+
|
|
42
|
+
for col in out.columns:
|
|
43
|
+
s = out[col]
|
|
44
|
+
if pd.api.types.is_bool_dtype(s):
|
|
45
|
+
continue # already 1 byte; nothing to gain
|
|
46
|
+
elif pd.api.types.is_integer_dtype(s):
|
|
47
|
+
out[col] = pd.to_numeric(s, downcast="integer")
|
|
48
|
+
elif pd.api.types.is_float_dtype(s):
|
|
49
|
+
out[col] = pd.to_numeric(s, downcast="float")
|
|
50
|
+
elif pd.api.types.is_object_dtype(s) or pd.api.types.is_string_dtype(s):
|
|
51
|
+
# Covers both classic ``object`` columns and the pandas>=3 ``str`` dtype.
|
|
52
|
+
n = len(s)
|
|
53
|
+
if n and s.nunique(dropna=False) / n < categorical_ratio:
|
|
54
|
+
out[col] = s.astype("category")
|
|
55
|
+
|
|
56
|
+
after = memory_usage_bytes(out)
|
|
57
|
+
if before:
|
|
58
|
+
logger.debug(
|
|
59
|
+
"Memory optimised: %.2f MB -> %.2f MB (%.1f%% saved)",
|
|
60
|
+
before / 1e6,
|
|
61
|
+
after / 1e6,
|
|
62
|
+
100 * (before - after) / before,
|
|
63
|
+
)
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def memory_report(df: pd.DataFrame) -> Dict[str, float]:
|
|
68
|
+
"""Return a small dict summarising memory footprint (in MB) per dtype family."""
|
|
69
|
+
usage = df.memory_usage(deep=True)
|
|
70
|
+
per_dtype: Dict[str, float] = {}
|
|
71
|
+
for col in df.columns:
|
|
72
|
+
family = str(df[col].dtype)
|
|
73
|
+
per_dtype[family] = per_dtype.get(family, 0.0) + float(usage[col]) / 1e6
|
|
74
|
+
per_dtype["total"] = float(usage.sum()) / 1e6
|
|
75
|
+
return {k: round(v, 4) for k, v in per_dtype.items()}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def estimate_savings(df: pd.DataFrame) -> float:
|
|
79
|
+
"""Return the fraction of memory that :func:`optimize_memory` would save."""
|
|
80
|
+
before = memory_usage_bytes(df)
|
|
81
|
+
if not before:
|
|
82
|
+
return 0.0
|
|
83
|
+
after = memory_usage_bytes(optimize_memory(df, copy=True))
|
|
84
|
+
return round((before - after) / before, 4)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Smart data readers with automatic detection and memory optimisation.
|
|
2
|
+
|
|
3
|
+
Each public ``read_*`` function returns a plain :class:`pandas.DataFrame`. The
|
|
4
|
+
facade layer is responsible for installing the result into the active session;
|
|
5
|
+
keeping these functions side-effect-free makes them trivially testable.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Optional
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
from ..core.config import DEFAULT_CONFIG, Config
|
|
16
|
+
from ..core.exceptions import DataLoadError
|
|
17
|
+
from ..core.logging import get_logger
|
|
18
|
+
from ..core.optional import optional_import
|
|
19
|
+
from ..core.types import PathLike
|
|
20
|
+
from .detect import coerce_datetimes, detect_delimiter, detect_encoding
|
|
21
|
+
from .optimize import optimize_memory
|
|
22
|
+
|
|
23
|
+
logger = get_logger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _post_process(
|
|
27
|
+
df: pd.DataFrame,
|
|
28
|
+
*,
|
|
29
|
+
config: Config,
|
|
30
|
+
parse_dates: bool,
|
|
31
|
+
) -> pd.DataFrame:
|
|
32
|
+
"""Shared post-load steps: datetime coercion + optional memory optimisation."""
|
|
33
|
+
if parse_dates:
|
|
34
|
+
df = coerce_datetimes(df)
|
|
35
|
+
if config.memory_optimize:
|
|
36
|
+
df = optimize_memory(df, copy=False)
|
|
37
|
+
return df
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def read_csv(
|
|
41
|
+
path: PathLike,
|
|
42
|
+
*,
|
|
43
|
+
config: Config = DEFAULT_CONFIG,
|
|
44
|
+
encoding: Optional[str] = None,
|
|
45
|
+
delimiter: Optional[str] = None,
|
|
46
|
+
parse_dates: bool = True,
|
|
47
|
+
**kwargs: Any,
|
|
48
|
+
) -> pd.DataFrame:
|
|
49
|
+
"""Read a CSV/TSV file, auto-detecting encoding and delimiter.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
path:
|
|
54
|
+
Filesystem path to the delimited text file.
|
|
55
|
+
encoding, delimiter:
|
|
56
|
+
Override auto-detection when provided.
|
|
57
|
+
parse_dates:
|
|
58
|
+
Auto-detect and convert datetime-like string columns.
|
|
59
|
+
**kwargs:
|
|
60
|
+
Forwarded to :func:`pandas.read_csv`.
|
|
61
|
+
"""
|
|
62
|
+
p = Path(path)
|
|
63
|
+
if not p.exists():
|
|
64
|
+
raise DataLoadError(f"File not found: {p}")
|
|
65
|
+
|
|
66
|
+
enc = encoding or detect_encoding(str(p))
|
|
67
|
+
sep = delimiter or detect_delimiter(str(p), encoding=enc)
|
|
68
|
+
logger.info("Reading CSV %s (encoding=%s, delimiter=%r)", p.name, enc, sep)
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
df = pd.read_csv(p, encoding=enc, sep=sep, **kwargs)
|
|
72
|
+
except Exception as exc: # noqa: BLE001 - re-raised as library error
|
|
73
|
+
raise DataLoadError(f"Failed to read CSV {p}: {exc}") from exc
|
|
74
|
+
|
|
75
|
+
return _post_process(df, config=config, parse_dates=parse_dates)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def read_excel(
|
|
79
|
+
path: PathLike,
|
|
80
|
+
*,
|
|
81
|
+
config: Config = DEFAULT_CONFIG,
|
|
82
|
+
sheet_name: Any = 0,
|
|
83
|
+
parse_dates: bool = True,
|
|
84
|
+
**kwargs: Any,
|
|
85
|
+
) -> pd.DataFrame:
|
|
86
|
+
"""Read an Excel workbook (requires the ``io`` extra / ``openpyxl``)."""
|
|
87
|
+
optional_import("openpyxl", extra="io")
|
|
88
|
+
p = Path(path)
|
|
89
|
+
if not p.exists():
|
|
90
|
+
raise DataLoadError(f"File not found: {p}")
|
|
91
|
+
logger.info("Reading Excel %s (sheet=%s)", p.name, sheet_name)
|
|
92
|
+
try:
|
|
93
|
+
df = pd.read_excel(p, sheet_name=sheet_name, **kwargs)
|
|
94
|
+
except Exception as exc: # noqa: BLE001
|
|
95
|
+
raise DataLoadError(f"Failed to read Excel {p}: {exc}") from exc
|
|
96
|
+
if isinstance(df, dict): # multiple sheets requested
|
|
97
|
+
return {k: _post_process(v, config=config, parse_dates=parse_dates) for k, v in df.items()} # type: ignore[return-value]
|
|
98
|
+
return _post_process(df, config=config, parse_dates=parse_dates)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def read_json(
|
|
102
|
+
path: PathLike,
|
|
103
|
+
*,
|
|
104
|
+
config: Config = DEFAULT_CONFIG,
|
|
105
|
+
parse_dates: bool = True,
|
|
106
|
+
**kwargs: Any,
|
|
107
|
+
) -> pd.DataFrame:
|
|
108
|
+
"""Read a JSON file into a dataframe (records or column orientation)."""
|
|
109
|
+
p = Path(path)
|
|
110
|
+
if not p.exists():
|
|
111
|
+
raise DataLoadError(f"File not found: {p}")
|
|
112
|
+
logger.info("Reading JSON %s", p.name)
|
|
113
|
+
try:
|
|
114
|
+
df = pd.read_json(p, **kwargs)
|
|
115
|
+
except Exception as exc: # noqa: BLE001
|
|
116
|
+
raise DataLoadError(f"Failed to read JSON {p}: {exc}") from exc
|
|
117
|
+
return _post_process(df, config=config, parse_dates=parse_dates)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def read_parquet(
|
|
121
|
+
path: PathLike,
|
|
122
|
+
*,
|
|
123
|
+
config: Config = DEFAULT_CONFIG,
|
|
124
|
+
parse_dates: bool = False,
|
|
125
|
+
**kwargs: Any,
|
|
126
|
+
) -> pd.DataFrame:
|
|
127
|
+
"""Read a Parquet file (requires the ``io`` extra / ``pyarrow``).
|
|
128
|
+
|
|
129
|
+
Datetime coercion defaults to off because Parquet already carries dtypes.
|
|
130
|
+
"""
|
|
131
|
+
optional_import("pyarrow", extra="io")
|
|
132
|
+
p = Path(path)
|
|
133
|
+
if not p.exists():
|
|
134
|
+
raise DataLoadError(f"File not found: {p}")
|
|
135
|
+
logger.info("Reading Parquet %s", p.name)
|
|
136
|
+
try:
|
|
137
|
+
df = pd.read_parquet(p, **kwargs)
|
|
138
|
+
except Exception as exc: # noqa: BLE001
|
|
139
|
+
raise DataLoadError(f"Failed to read Parquet {p}: {exc}") from exc
|
|
140
|
+
return _post_process(df, config=config, parse_dates=parse_dates)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def read_sql(
|
|
144
|
+
query: str,
|
|
145
|
+
con: Any,
|
|
146
|
+
*,
|
|
147
|
+
config: Config = DEFAULT_CONFIG,
|
|
148
|
+
parse_dates: bool = True,
|
|
149
|
+
**kwargs: Any,
|
|
150
|
+
) -> pd.DataFrame:
|
|
151
|
+
"""Run ``query`` against a SQL connection/engine and return the result.
|
|
152
|
+
|
|
153
|
+
``con`` may be a DBAPI connection or a SQLAlchemy engine/URL string.
|
|
154
|
+
"""
|
|
155
|
+
logger.info("Reading SQL query")
|
|
156
|
+
try:
|
|
157
|
+
df = pd.read_sql(query, con, **kwargs)
|
|
158
|
+
except Exception as exc: # noqa: BLE001
|
|
159
|
+
raise DataLoadError(f"Failed to execute SQL read: {exc}") from exc
|
|
160
|
+
return _post_process(df, config=config, parse_dates=parse_dates)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Murthylytics modeling module (roadmap — see project README for status)."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Murthylytics pipeline module (roadmap — see project README for status)."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Murthylytics plugins module (roadmap — see project README for status)."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Murthylytics preprocess module (roadmap — see project README for status)."""
|
murthylytics/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Murthylytics viz module (roadmap — see project README for status)."""
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: murthylytics
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Production-grade, AI-powered machine learning data-analysis and preparation library.
|
|
5
|
+
Project-URL: Homepage, https://github.com/pranavmurthy/murthylytics
|
|
6
|
+
Project-URL: Repository, https://github.com/pranavmurthy/murthylytics
|
|
7
|
+
Project-URL: Issues, https://github.com/pranavmurthy/murthylytics/issues
|
|
8
|
+
Author: Pranav Murthy
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: autoeda,automl,data-cleaning,data-science,eda,feature-engineering,feature-selection,machine-learning,preprocessing
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Requires-Dist: numpy>=1.23
|
|
25
|
+
Requires-Dist: pandas>=1.5
|
|
26
|
+
Provides-Extra: all
|
|
27
|
+
Requires-Dist: charset-normalizer>=3.0; extra == 'all'
|
|
28
|
+
Requires-Dist: jinja2>=3.1; extra == 'all'
|
|
29
|
+
Requires-Dist: lightgbm>=3.3; extra == 'all'
|
|
30
|
+
Requires-Dist: matplotlib>=3.6; extra == 'all'
|
|
31
|
+
Requires-Dist: openpyxl>=3.1; extra == 'all'
|
|
32
|
+
Requires-Dist: pyarrow>=10.0; extra == 'all'
|
|
33
|
+
Requires-Dist: scikit-learn>=1.1; extra == 'all'
|
|
34
|
+
Requires-Dist: scipy>=1.9; extra == 'all'
|
|
35
|
+
Requires-Dist: seaborn>=0.12; extra == 'all'
|
|
36
|
+
Requires-Dist: sqlalchemy>=2.0; extra == 'all'
|
|
37
|
+
Requires-Dist: xgboost>=1.7; extra == 'all'
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: black>=23.0; extra == 'dev'
|
|
40
|
+
Requires-Dist: mypy>=1.5; extra == 'dev'
|
|
41
|
+
Requires-Dist: pre-commit>=3.4; extra == 'dev'
|
|
42
|
+
Requires-Dist: pytest-cov>=4.1; extra == 'dev'
|
|
43
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
44
|
+
Requires-Dist: ruff>=0.1; extra == 'dev'
|
|
45
|
+
Provides-Extra: io
|
|
46
|
+
Requires-Dist: charset-normalizer>=3.0; extra == 'io'
|
|
47
|
+
Requires-Dist: openpyxl>=3.1; extra == 'io'
|
|
48
|
+
Requires-Dist: pyarrow>=10.0; extra == 'io'
|
|
49
|
+
Requires-Dist: sqlalchemy>=2.0; extra == 'io'
|
|
50
|
+
Provides-Extra: ml
|
|
51
|
+
Requires-Dist: lightgbm>=3.3; extra == 'ml'
|
|
52
|
+
Requires-Dist: xgboost>=1.7; extra == 'ml'
|
|
53
|
+
Provides-Extra: polars
|
|
54
|
+
Requires-Dist: polars>=0.20; extra == 'polars'
|
|
55
|
+
Provides-Extra: report
|
|
56
|
+
Requires-Dist: jinja2>=3.1; extra == 'report'
|
|
57
|
+
Provides-Extra: stats
|
|
58
|
+
Requires-Dist: scikit-learn>=1.1; extra == 'stats'
|
|
59
|
+
Requires-Dist: scipy>=1.9; extra == 'stats'
|
|
60
|
+
Provides-Extra: viz
|
|
61
|
+
Requires-Dist: matplotlib>=3.6; extra == 'viz'
|
|
62
|
+
Requires-Dist: seaborn>=0.12; extra == 'viz'
|
|
63
|
+
Description-Content-Type: text/markdown
|
|
64
|
+
|
|
65
|
+
# Murthylytics
|
|
66
|
+
|
|
67
|
+
**Production-grade, AI-powered machine-learning data analysis & preparation for Python.**
|
|
68
|
+
|
|
69
|
+
Murthylytics turns hundreds of lines of exploratory-data-analysis and preprocessing
|
|
70
|
+
boilerplate into a handful of intuitive commands — while staying transparent,
|
|
71
|
+
configurable and production-quality. Think of it as a next-generation companion to
|
|
72
|
+
pandas, ydata-profiling, Sweetviz and scikit-learn's preprocessing tools, unified
|
|
73
|
+
behind one stateful API.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
import murthylytics as mlt
|
|
77
|
+
|
|
78
|
+
mlt.read_csv("data.csv") # auto-detects encoding, delimiter, dtypes, datetimes
|
|
79
|
+
|
|
80
|
+
mlt.summary() # compact overview
|
|
81
|
+
mlt.inspect() # intelligent profiling: roles, targets, leakage, issues
|
|
82
|
+
|
|
83
|
+
mlt.clean_data() # roadmap: intelligent cleaning & imputation
|
|
84
|
+
mlt.auto_eda() # roadmap: HTML / Markdown / PDF / JSON reports
|
|
85
|
+
mlt.preprocess() # roadmap: task detection + preprocessing pipeline
|
|
86
|
+
mlt.recommend_model() # roadmap: model recommendations with justification
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Why Murthylytics
|
|
90
|
+
|
|
91
|
+
- **One stateful facade.** Load a dataset once; every call operates on the active
|
|
92
|
+
session — no handle passing.
|
|
93
|
+
- **Intelligent by default.** Encoding/delimiter/dtype detection, column-role
|
|
94
|
+
inference, target detection, class-imbalance and data-leakage checks, all automatic.
|
|
95
|
+
- **Lightweight core, optional power.** Base install is just pandas + numpy. Heavy
|
|
96
|
+
tooling (plots, XGBoost, SHAP, Parquet…) is lazy-loaded behind extras.
|
|
97
|
+
- **Engineered to last.** `src/` layout, full type hints, SOLID modules, a pure
|
|
98
|
+
stateless engine under the facade, and a test suite.
|
|
99
|
+
|
|
100
|
+
## Installation
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
pip install murthylytics # core (pandas + numpy)
|
|
104
|
+
pip install 'murthylytics[stats]' # scipy + scikit-learn powered analysis
|
|
105
|
+
pip install 'murthylytics[viz]' # matplotlib + seaborn plotting
|
|
106
|
+
pip install 'murthylytics[all]' # everything
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
> Develop locally: `pip install -e '.[dev,stats]'`
|
|
110
|
+
|
|
111
|
+
## Quick start
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
import murthylytics as mlt
|
|
115
|
+
|
|
116
|
+
mlt.read_csv("titanic.csv")
|
|
117
|
+
|
|
118
|
+
mlt.shape() # {'rows': 891, 'columns': 12}
|
|
119
|
+
mlt.missing() # per-column missing counts & %
|
|
120
|
+
report = mlt.inspect()
|
|
121
|
+
print(report) # human-readable summary
|
|
122
|
+
print(report.recommendations) # actionable next steps
|
|
123
|
+
report.to_dict() # JSON-serialisable
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Feature status
|
|
127
|
+
|
|
128
|
+
| Area | API | Status |
|
|
129
|
+
|------|-----|--------|
|
|
130
|
+
| Data I/O (CSV/Excel/JSON/Parquet/SQL) | `read_*` | ✅ available |
|
|
131
|
+
| Exploration | `show/top/bottom/shape/columns/summary/describe/info/types/missing/duplicates/memory` | ✅ available |
|
|
132
|
+
| Intelligent inspection | `inspect` | ✅ available |
|
|
133
|
+
| Intelligent cleaning | `clean_data` | 🚧 roadmap (iteration 2) |
|
|
134
|
+
| Visualization | `cleaned_pyplot` | 🚧 roadmap (iteration 3) |
|
|
135
|
+
| AutoEDA reports | `auto_eda` | 🚧 roadmap (iteration 4) |
|
|
136
|
+
| Preprocessing | `preprocess` | 🚧 roadmap (iteration 5) |
|
|
137
|
+
| Feature engineering / selection | `engineer_features` / `select_features` | 🚧 roadmap (iteration 6) |
|
|
138
|
+
| Model recommendation | `recommend_model` | 🚧 roadmap (iteration 7) |
|
|
139
|
+
| Pipeline export | `export_pipeline` | 🚧 roadmap (iteration 8) |
|
|
140
|
+
| Plugins (malware, finance, …) | `use_plugin` | 🚧 roadmap (iteration 9) |
|
|
141
|
+
|
|
142
|
+
The roadmap functions are already part of the public surface with stable
|
|
143
|
+
signatures; calling one today raises a clear `NotImplementedError` pointing at the
|
|
144
|
+
iteration that delivers it.
|
|
145
|
+
|
|
146
|
+
## Architecture
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
murthylytics/
|
|
150
|
+
├── core/ config · session/context · logging · exceptions · types
|
|
151
|
+
├── io/ smart readers · encoding/delimiter/dtype detection · memory opt
|
|
152
|
+
├── exploration/ display helpers · intelligent inspection
|
|
153
|
+
├── cleaning/ (roadmap) duplicates · imputation · outliers · text fixes
|
|
154
|
+
├── viz/ (roadmap) auto-plot dispatcher
|
|
155
|
+
├── eda/ (roadmap) HTML/MD/PDF/JSON reports
|
|
156
|
+
├── preprocess/ (roadmap) task detection · scaling · encoding · pipelines
|
|
157
|
+
├── features/ (roadmap) engineering · selection
|
|
158
|
+
├── modeling/ (roadmap) model recommendation
|
|
159
|
+
├── pipeline/ (roadmap) save/load/export
|
|
160
|
+
└── plugins/ (roadmap) domain plugins
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
All mutable state lives in a single `Session`; the engine modules are pure
|
|
164
|
+
functions, which keeps them independently testable and reusable.
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT © Pranav Murthy
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
murthylytics/__init__.py,sha256=qQEfnT-aEpEF_-yU77tSoDknQSN08HX-KS0sLGaBOEU,8715
|
|
2
|
+
murthylytics/_version.py,sha256=Y_2bqYDoRNq0PnhUrLbcwDTa_O-tiNwjoBWEyVHCbyk,97
|
|
3
|
+
murthylytics/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
murthylytics/cleaning/__init__.py,sha256=uiCJNBp5Gi4hCkRnqihutGY-TOwoaCRHs4BTYkESR2Q,80
|
|
5
|
+
murthylytics/core/__init__.py,sha256=3rsJODb7lgPBMUppzAnPnzwLrMc9e97tN39nwdBWh9c,1034
|
|
6
|
+
murthylytics/core/config.py,sha256=GO6US631lZCLjnw_DsQ9b_SITCKmZE6_d4pworuNvrk,3410
|
|
7
|
+
murthylytics/core/context.py,sha256=vy0AUM57G6yS01sq2tjVI_56PZ5mhOhSdMY8JqFT4vE,4033
|
|
8
|
+
murthylytics/core/exceptions.py,sha256=MGeJ_x4-e0UilBNZT6e90zM1D8n6urvCKIPS5XdFL-g,1721
|
|
9
|
+
murthylytics/core/logging.py,sha256=6lTS3C8X00ofAxPwqOVRwJrLMgif53CgLyPvkebUF68,2436
|
|
10
|
+
murthylytics/core/optional.py,sha256=1xXOzGVO5tIh_ErYgGeT10CFd-ep_NyS_qn4TijmLyY,1053
|
|
11
|
+
murthylytics/core/types.py,sha256=3hDVsmfyTnM38-qo3SJ7DBkW_AsG_YhGC91423cScpc,1265
|
|
12
|
+
murthylytics/eda/__init__.py,sha256=B-eWo5TFQuiIsFfg-JrHsQq-L5PtAhcICK4uRgXK9Z0,75
|
|
13
|
+
murthylytics/exploration/__init__.py,sha256=Yeoh1C-y8NwpBFESuULpf2q4JnGn6caGerMRqZano1A,577
|
|
14
|
+
murthylytics/exploration/display.py,sha256=mmz35Baul1X7ivPuJ697uwquY1KidvHLoiI9rI9tRn4,3288
|
|
15
|
+
murthylytics/exploration/inspect.py,sha256=3n_E3EnoH_VMVItQs83gkZOi_KMm-y1FnthYJkFvd1k,12980
|
|
16
|
+
murthylytics/features/__init__.py,sha256=L7-4zxRwVQhTSRdde-OPNaK8chqSiH-dDws9_Y16eu8,80
|
|
17
|
+
murthylytics/io/__init__.py,sha256=-Kd4tc3dRep4mYPWz4FGtaj5f-WJrz_xH_1FllUW5ls,696
|
|
18
|
+
murthylytics/io/detect.py,sha256=WjSKTHaAEgRPp71AUxb_ue2V1Lo0QdQi5DvA5H44LBo,4151
|
|
19
|
+
murthylytics/io/optimize.py,sha256=qPh4t6q4VcCvsuxh6BYW-yskZgJtk1e7C0AkpFfOHnk,2739
|
|
20
|
+
murthylytics/io/readers.py,sha256=wjsrEj_cPyNi_xTBc3HmU7g2mpiqPmKtwMZfzI4OI4I,5115
|
|
21
|
+
murthylytics/modeling/__init__.py,sha256=3BFFE78lOU3Jd7aWulae7yOAf983lLElmRjXdYsI1gQ,80
|
|
22
|
+
murthylytics/pipeline/__init__.py,sha256=UPTr4p3SCa-rdE4tpSQFY7y9utpE6izSVXcO34jyofE,80
|
|
23
|
+
murthylytics/plugins/__init__.py,sha256=gvoGZiplIS_5d_C60QZ8UzLnagttzCZqDYQB9Ou6C-E,79
|
|
24
|
+
murthylytics/preprocess/__init__.py,sha256=6MNf_fVnFDF8amNI8i-a7pFbbITqDx57XmuU_iz2OGE,82
|
|
25
|
+
murthylytics/viz/__init__.py,sha256=fQbkkGKmXMZh1iNKnSUTfCG9v9CaUGPGSSwcqsPLY-A,75
|
|
26
|
+
murthylytics-0.1.0.dist-info/METADATA,sha256=0sYSh-4CX1l0ZatcpWR19y9GLBwkjqUSSkuBDGH9Exg,7075
|
|
27
|
+
murthylytics-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
28
|
+
murthylytics-0.1.0.dist-info/licenses/LICENSE,sha256=uXZ_CuyPB27fqGyOVYgix7vgrdrSRYJX6BZH7afbB64,1070
|
|
29
|
+
murthylytics-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pranav Murthy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|