tlf-correlation-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tlf_correlation_engine/__init__.py +28 -0
- tlf_correlation_engine/engine.py +155 -0
- tlf_correlation_engine/errors.py +14 -0
- tlf_correlation_engine/interactive.py +83 -0
- tlf_correlation_engine/loader.py +95 -0
- tlf_correlation_engine/main.py +135 -0
- tlf_correlation_engine/methods.py +74 -0
- tlf_correlation_engine-0.1.0.dist-info/METADATA +138 -0
- tlf_correlation_engine-0.1.0.dist-info/RECORD +13 -0
- tlf_correlation_engine-0.1.0.dist-info/WHEEL +5 -0
- tlf_correlation_engine-0.1.0.dist-info/entry_points.txt +2 -0
- tlf_correlation_engine-0.1.0.dist-info/licenses/LICENSE +9 -0
- tlf_correlation_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tlf-correlation-engine
|
|
3
|
+
-----------------------
|
|
4
|
+
Country-agnostic correlation analysis (Pearson, Spearman, Kendall) over
|
|
5
|
+
any pandas DataFrame's numeric columns — coefficient matrices, p-values,
|
|
6
|
+
and a sortable/filterable long-form report of relationship strength.
|
|
7
|
+
Part of the TLF-Data-Analysis repo, part of the TLF ("The Living Facts")
|
|
8
|
+
initiative.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from .engine import CorrelationEngine, SUPPORTED_METHODS
|
|
12
|
+
from .methods import pairwise_correlation
|
|
13
|
+
from .loader import TabularLoader, UnsupportedFileError
|
|
14
|
+
from .errors import CorrelationEngineError, InvalidMethodError, InsufficientDataError
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"__version__",
|
|
18
|
+
"CorrelationEngine",
|
|
19
|
+
"SUPPORTED_METHODS",
|
|
20
|
+
"pairwise_correlation",
|
|
21
|
+
"TabularLoader",
|
|
22
|
+
"UnsupportedFileError",
|
|
23
|
+
"CorrelationEngineError",
|
|
24
|
+
"InvalidMethodError",
|
|
25
|
+
"InsufficientDataError",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core CorrelationEngine — country-agnostic correlation analysis over any
|
|
3
|
+
pandas DataFrame's numeric columns.
|
|
4
|
+
|
|
5
|
+
Unlike a bare ``df.corr()`` call, this engine also reports statistical
|
|
6
|
+
significance (p-values) and the number of paired observations behind
|
|
7
|
+
each coefficient, and can produce a sorted, filterable long-form report
|
|
8
|
+
of the strongest / most significant relationships in a dataset.
|
|
9
|
+
|
|
10
|
+
matrix(), pvalue_matrix(), pairwise(), and report() are all backed by
|
|
11
|
+
the same MIN_OBSERVATIONS-aware pairwise_correlation() calls (see
|
|
12
|
+
methods.py), so they always agree with each other: a pair with too few
|
|
13
|
+
paired observations to be meaningful comes back as NaN everywhere, not
|
|
14
|
+
just in the p-value output.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from .errors import InvalidMethodError, InsufficientDataError
|
|
20
|
+
from .methods import METHODS, pairwise_correlation
|
|
21
|
+
|
|
22
|
+
SUPPORTED_METHODS = tuple(METHODS)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class CorrelationEngine:
|
|
26
|
+
"""Compute correlations across a DataFrame's numeric columns.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
df : pandas.DataFrame
|
|
31
|
+
Any DataFrame. Only numeric columns are used; non-numeric
|
|
32
|
+
columns are ignored automatically.
|
|
33
|
+
method : str
|
|
34
|
+
One of "pearson" (default), "spearman", or "kendall".
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self, df: pd.DataFrame, method: str = "pearson"):
|
|
38
|
+
if method not in SUPPORTED_METHODS:
|
|
39
|
+
raise InvalidMethodError(
|
|
40
|
+
f"Unsupported method '{method}'. Choose from: {', '.join(SUPPORTED_METHODS)}."
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
self.method = method
|
|
44
|
+
self.df = df
|
|
45
|
+
self._numeric_df = df.select_dtypes(include="number")
|
|
46
|
+
self._grid_cache = None # set on first matrix()/pvalue_matrix() call
|
|
47
|
+
|
|
48
|
+
if self._numeric_df.shape[1] < 2:
|
|
49
|
+
raise InsufficientDataError(
|
|
50
|
+
"Need at least 2 numeric columns to compute correlations; "
|
|
51
|
+
f"found {self._numeric_df.shape[1]}."
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def columns(self):
|
|
56
|
+
"""Numeric columns considered by this engine."""
|
|
57
|
+
return list(self._numeric_df.columns)
|
|
58
|
+
|
|
59
|
+
def _pairwise_grid(self):
|
|
60
|
+
"""Computes the coefficient and p-value matrices in one pass, both
|
|
61
|
+
going through the same MIN_OBSERVATIONS-aware pairwise_correlation()
|
|
62
|
+
used by pairwise()/report() — so a pair with too few paired
|
|
63
|
+
observations is NaN everywhere, not just in the p-value matrix.
|
|
64
|
+
Cached per-instance since method/data don't change after __init__.
|
|
65
|
+
"""
|
|
66
|
+
if self._grid_cache is not None:
|
|
67
|
+
return self._grid_cache
|
|
68
|
+
|
|
69
|
+
cols = self.columns
|
|
70
|
+
coef = pd.DataFrame(index=cols, columns=cols, dtype=float)
|
|
71
|
+
pval = pd.DataFrame(index=cols, columns=cols, dtype=float)
|
|
72
|
+
|
|
73
|
+
for i, col_a in enumerate(cols):
|
|
74
|
+
# A column is perfectly, trivially correlated with itself,
|
|
75
|
+
# regardless of n — this is a definitional identity, not a
|
|
76
|
+
# statistical claim, so MIN_OBSERVATIONS doesn't apply here.
|
|
77
|
+
coef.loc[col_a, col_a] = 1.0
|
|
78
|
+
pval.loc[col_a, col_a] = 0.0
|
|
79
|
+
for col_b in cols[i + 1:]:
|
|
80
|
+
result = pairwise_correlation(
|
|
81
|
+
self._numeric_df[col_a], self._numeric_df[col_b], method=self.method
|
|
82
|
+
)
|
|
83
|
+
coef.loc[col_a, col_b] = result["coefficient"]
|
|
84
|
+
coef.loc[col_b, col_a] = result["coefficient"]
|
|
85
|
+
pval.loc[col_a, col_b] = result["p_value"]
|
|
86
|
+
pval.loc[col_b, col_a] = result["p_value"]
|
|
87
|
+
|
|
88
|
+
self._grid_cache = (coef, pval)
|
|
89
|
+
return self._grid_cache
|
|
90
|
+
|
|
91
|
+
def matrix(self) -> pd.DataFrame:
|
|
92
|
+
"""Correlation coefficient matrix. A pair with fewer than
|
|
93
|
+
MIN_OBSERVATIONS paired observations (see methods.py) is NaN,
|
|
94
|
+
matching pairwise()/report() — this is NOT the same as a bare
|
|
95
|
+
``df.corr()`` call, which would compute a (statistically
|
|
96
|
+
meaningless) coefficient from as few as 2 points."""
|
|
97
|
+
coef, _ = self._pairwise_grid()
|
|
98
|
+
return coef
|
|
99
|
+
|
|
100
|
+
def pvalue_matrix(self) -> pd.DataFrame:
|
|
101
|
+
"""P-value matrix aligned with ``matrix()``. Diagonal is 0.0 (a
|
|
102
|
+
column is perfectly, trivially correlated with itself)."""
|
|
103
|
+
_, pval = self._pairwise_grid()
|
|
104
|
+
return pval
|
|
105
|
+
|
|
106
|
+
def pairwise(self, col_a: str, col_b: str) -> dict:
|
|
107
|
+
"""Correlation between two specific columns, with n and p-value."""
|
|
108
|
+
return pairwise_correlation(
|
|
109
|
+
self._numeric_df[col_a], self._numeric_df[col_b], method=self.method
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def report(self, threshold: float = None, significant_only: bool = False, alpha: float = 0.05) -> pd.DataFrame:
|
|
113
|
+
"""Long-form report: one row per unique column pair, sorted by
|
|
114
|
+
the strength of the relationship (descending |coefficient|).
|
|
115
|
+
|
|
116
|
+
Parameters
|
|
117
|
+
----------
|
|
118
|
+
threshold : float, optional
|
|
119
|
+
Only include pairs where |coefficient| >= threshold.
|
|
120
|
+
significant_only : bool
|
|
121
|
+
Only include pairs where p_value < alpha.
|
|
122
|
+
alpha : float
|
|
123
|
+
Significance threshold used when significant_only=True.
|
|
124
|
+
"""
|
|
125
|
+
cols = self.columns
|
|
126
|
+
rows = []
|
|
127
|
+
|
|
128
|
+
for i, col_a in enumerate(cols):
|
|
129
|
+
for col_b in cols[i + 1:]:
|
|
130
|
+
result = self.pairwise(col_a, col_b)
|
|
131
|
+
rows.append({
|
|
132
|
+
"variable_1": col_a,
|
|
133
|
+
"variable_2": col_b,
|
|
134
|
+
"method": result["method"],
|
|
135
|
+
"coefficient": result["coefficient"],
|
|
136
|
+
"p_value": result["p_value"],
|
|
137
|
+
"n": result["n"],
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
report_df = pd.DataFrame(rows)
|
|
141
|
+
|
|
142
|
+
if report_df.empty:
|
|
143
|
+
return report_df
|
|
144
|
+
|
|
145
|
+
if threshold is not None:
|
|
146
|
+
report_df = report_df[report_df["coefficient"].abs() >= threshold]
|
|
147
|
+
|
|
148
|
+
if significant_only:
|
|
149
|
+
report_df = report_df[report_df["p_value"] < alpha]
|
|
150
|
+
|
|
151
|
+
report_df = report_df.reindex(
|
|
152
|
+
report_df["coefficient"].abs().sort_values(ascending=False).index
|
|
153
|
+
).reset_index(drop=True)
|
|
154
|
+
|
|
155
|
+
return report_df
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Exceptions raised by tlf_correlation_engine."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class CorrelationEngineError(Exception):
|
|
5
|
+
"""Base exception for tlf_correlation_engine."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class InvalidMethodError(CorrelationEngineError):
|
|
9
|
+
"""Raised when an unsupported correlation method is requested."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class InsufficientDataError(CorrelationEngineError):
|
|
13
|
+
"""Raised when a DataFrame doesn't have enough numeric columns or rows
|
|
14
|
+
to compute a meaningful correlation."""
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""
|
|
2
|
+
interactive.py — prompts
|
|
3
|
+
Terminal prompts for any run configuration the user didn't supply via
|
|
4
|
+
CLI flags. Flags always win; a prompt only fires for whatever's still
|
|
5
|
+
missing after argument parsing. In --yes (non-interactive) mode, none
|
|
6
|
+
of these are called at all — main.py raises a clear error instead if
|
|
7
|
+
something required is still missing.
|
|
8
|
+
|
|
9
|
+
Uses questionary for arrow-key menus and multi-select checklists.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import questionary
|
|
13
|
+
|
|
14
|
+
from .engine import SUPPORTED_METHODS
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def prompt_for_path() -> str:
|
|
18
|
+
return questionary.path("Path to your data file (CSV, Excel, or JSON):").ask()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def prompt_for_sheet(sheet_names: list):
|
|
22
|
+
if not sheet_names or len(sheet_names) == 1:
|
|
23
|
+
return sheet_names[0] if sheet_names else None
|
|
24
|
+
return questionary.select("Which sheet should be read?", choices=sheet_names).ask()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def prompt_for_method() -> str:
|
|
28
|
+
return questionary.select(
|
|
29
|
+
"Which correlation method?",
|
|
30
|
+
choices=[
|
|
31
|
+
questionary.Choice("Pearson (linear relationships)", value="pearson"),
|
|
32
|
+
questionary.Choice("Spearman (monotonic, rank-based, robust to outliers)", value="spearman"),
|
|
33
|
+
questionary.Choice("Kendall (rank concordance, good for small samples)", value="kendall"),
|
|
34
|
+
],
|
|
35
|
+
).ask()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def prompt_for_output() -> str:
|
|
39
|
+
return questionary.select(
|
|
40
|
+
"What should be shown?",
|
|
41
|
+
choices=[
|
|
42
|
+
questionary.Choice("Long-form report (sorted by relationship strength)", value="report"),
|
|
43
|
+
questionary.Choice("Correlation coefficient matrix", value="matrix"),
|
|
44
|
+
questionary.Choice("P-value matrix", value="pvalues"),
|
|
45
|
+
],
|
|
46
|
+
).ask()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def prompt_for_report_filters() -> tuple:
|
|
50
|
+
"""Only asked when output == 'report'. Returns (threshold, significant_only)."""
|
|
51
|
+
significant_only = questionary.confirm(
|
|
52
|
+
"Only show statistically significant pairs (p < 0.05)?", default=False
|
|
53
|
+
).ask()
|
|
54
|
+
|
|
55
|
+
use_threshold = questionary.confirm(
|
|
56
|
+
"Filter by minimum correlation strength?", default=False
|
|
57
|
+
).ask()
|
|
58
|
+
threshold = None
|
|
59
|
+
if use_threshold:
|
|
60
|
+
raw = questionary.text("Minimum |coefficient| (0.0–1.0):", default="0.5").ask()
|
|
61
|
+
threshold = float(raw)
|
|
62
|
+
|
|
63
|
+
return threshold, significant_only
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def prompt_for_export() -> tuple:
|
|
67
|
+
# NOTE: questionary.Choice(value=None) is treated as "no value given"
|
|
68
|
+
# and defaults to the choice's *title* text, not to a real None. So
|
|
69
|
+
# "no export" needs its own distinct sentinel ("none") rather than
|
|
70
|
+
# value=None, or selecting it would be silently mistaken for a
|
|
71
|
+
# (nonsense) export format and still try to prompt for a path.
|
|
72
|
+
fmt = questionary.select(
|
|
73
|
+
"Export the result to a file?",
|
|
74
|
+
choices=[
|
|
75
|
+
questionary.Choice("No, just print to terminal", value="none"),
|
|
76
|
+
questionary.Choice("CSV", value="csv"),
|
|
77
|
+
questionary.Choice("JSON", value="json"),
|
|
78
|
+
],
|
|
79
|
+
).ask()
|
|
80
|
+
if fmt in (None, "none"):
|
|
81
|
+
return None, None
|
|
82
|
+
path = questionary.text(f"Export path (e.g. output.{fmt}):").ask()
|
|
83
|
+
return fmt, path
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""
|
|
2
|
+
loader.py — TabularLoader
|
|
3
|
+
Reads a CSV, Excel, or JSON file into a plain pandas DataFrame with no
|
|
4
|
+
schema assumptions: no required columns, no renaming. Column headers
|
|
5
|
+
are used exactly as given in the source file — CorrelationEngine picks
|
|
6
|
+
out the numeric ones itself.
|
|
7
|
+
|
|
8
|
+
This mirrors tlf-statistical-summary's TabularLoader (same supported
|
|
9
|
+
formats, same PDF rejection message) since both packages are meant to
|
|
10
|
+
sit downstream of already-cleaned data, not parse raw PDFs themselves.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from .errors import CorrelationEngineError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class UnsupportedFileError(CorrelationEngineError):
|
|
22
|
+
"""Raised when the input file's type can't be read at all (e.g. a
|
|
23
|
+
raw PDF, or an extension this package doesn't recognize)."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TabularLoader:
|
|
27
|
+
"""
|
|
28
|
+
Loads tabular data from a CSV, Excel, or JSON file, as-is.
|
|
29
|
+
|
|
30
|
+
Usage:
|
|
31
|
+
loader = TabularLoader("census_data.csv")
|
|
32
|
+
df = loader.load()
|
|
33
|
+
|
|
34
|
+
loader = TabularLoader("data.xlsx", sheet="District Summary")
|
|
35
|
+
df = loader.load()
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
SUPPORTED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".json"}
|
|
39
|
+
|
|
40
|
+
def __init__(self, filepath: str, sheet=None):
|
|
41
|
+
self.filepath = Path(filepath)
|
|
42
|
+
# Excel only: None = first/only sheet; a name or index restricts to one.
|
|
43
|
+
self.sheet = sheet
|
|
44
|
+
|
|
45
|
+
def load(self) -> pd.DataFrame:
|
|
46
|
+
if not self.filepath.exists():
|
|
47
|
+
raise FileNotFoundError(f"Data file not found: {self.filepath}")
|
|
48
|
+
return self._read_file()
|
|
49
|
+
|
|
50
|
+
def _read_file(self) -> pd.DataFrame:
|
|
51
|
+
ext = self.filepath.suffix.lower()
|
|
52
|
+
|
|
53
|
+
if ext == ".csv":
|
|
54
|
+
return pd.read_csv(self.filepath)
|
|
55
|
+
|
|
56
|
+
if ext in (".xlsx", ".xls"):
|
|
57
|
+
sheet = self.sheet if self.sheet is not None else 0
|
|
58
|
+
return pd.read_excel(self.filepath, sheet_name=sheet)
|
|
59
|
+
|
|
60
|
+
if ext == ".json":
|
|
61
|
+
return self._read_json()
|
|
62
|
+
|
|
63
|
+
if ext == ".pdf":
|
|
64
|
+
raise UnsupportedFileError(
|
|
65
|
+
f"Cannot read '{self.filepath.name}' directly: this package does not "
|
|
66
|
+
f"parse PDFs. Convert the PDF's tables into a CSV/Excel/JSON file "
|
|
67
|
+
f"first (e.g. with tlf-data-cleaning), then load that output here instead."
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
raise UnsupportedFileError(
|
|
71
|
+
f"Unsupported file type '{ext}' for '{self.filepath.name}'. "
|
|
72
|
+
f"This package reads: {sorted(self.SUPPORTED_EXTENSIONS)}."
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def _read_json(self) -> pd.DataFrame:
|
|
76
|
+
with open(self.filepath, "r", encoding="utf-8") as f:
|
|
77
|
+
data = json.load(f)
|
|
78
|
+
|
|
79
|
+
if isinstance(data, list):
|
|
80
|
+
return pd.DataFrame(data)
|
|
81
|
+
if isinstance(data, dict):
|
|
82
|
+
return pd.DataFrame([data])
|
|
83
|
+
raise UnsupportedFileError(
|
|
84
|
+
f"'{self.filepath.name}' is valid JSON but not in a shape this package "
|
|
85
|
+
f"can read: expected a list of row-objects or a single flat object, "
|
|
86
|
+
f"got {type(data).__name__}."
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def list_sheets(self) -> list:
|
|
90
|
+
"""Returns the sheet names in an Excel workbook, for building an
|
|
91
|
+
interactive sheet-selection prompt. Empty list for non-Excel files."""
|
|
92
|
+
ext = self.filepath.suffix.lower()
|
|
93
|
+
if ext not in (".xlsx", ".xls"):
|
|
94
|
+
return []
|
|
95
|
+
return pd.ExcelFile(self.filepath).sheet_names
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""
|
|
2
|
+
main.py — tlf-correlation-engine CLI
|
|
3
|
+
Country-agnostic Pearson/Spearman/Kendall correlation analysis for any
|
|
4
|
+
tabular file.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
python -m tlf_correlation_engine.main --data census.csv
|
|
8
|
+
python -m tlf_correlation_engine.main --data census.csv --method spearman --output matrix
|
|
9
|
+
python -m tlf_correlation_engine.main --data census.csv --yes --output report --export csv --export-path out.csv
|
|
10
|
+
|
|
11
|
+
Flags are read first; anything not supplied is filled in via
|
|
12
|
+
interactive terminal prompts, UNLESS --yes is passed, in which case
|
|
13
|
+
any required-but-missing value raises a clear error instead of
|
|
14
|
+
prompting (so this can run unattended, e.g. from a script or cron job,
|
|
15
|
+
without hanging on input()).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
import pandas as pd
|
|
22
|
+
|
|
23
|
+
from .loader import TabularLoader
|
|
24
|
+
from .engine import CorrelationEngine, SUPPORTED_METHODS
|
|
25
|
+
from .errors import CorrelationEngineError
|
|
26
|
+
from . import interactive
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_args():
|
|
30
|
+
parser = argparse.ArgumentParser(description="tlf-correlation-engine: Pearson/Spearman/Kendall correlation analysis")
|
|
31
|
+
parser.add_argument("--data", default=None, help="Path to a CSV, Excel, or JSON file.")
|
|
32
|
+
parser.add_argument("--sheet", default=None, help="Excel sheet name to read. Default: first sheet.")
|
|
33
|
+
parser.add_argument("--method", default=None, choices=SUPPORTED_METHODS, help="Correlation method.")
|
|
34
|
+
parser.add_argument("--output", default=None, choices=["report", "matrix", "pvalues"],
|
|
35
|
+
help="What to show: long-form report, coefficient matrix, or p-value matrix.")
|
|
36
|
+
parser.add_argument("--threshold", type=float, default=None,
|
|
37
|
+
help="Report only: minimum |coefficient| to include.")
|
|
38
|
+
parser.add_argument("--significant-only", dest="significant_only", action="store_true", default=None,
|
|
39
|
+
help="Report only: only include pairs with p < --alpha.")
|
|
40
|
+
parser.add_argument("--alpha", type=float, default=0.05, help="Significance threshold for --significant-only.")
|
|
41
|
+
parser.add_argument("--export", default=None, choices=["csv", "json"], help="Export format.")
|
|
42
|
+
parser.add_argument("--export-path", default=None, help="Export file path.")
|
|
43
|
+
parser.add_argument("--yes", action="store_true",
|
|
44
|
+
help="Non-interactive mode: never prompt, error on missing required values.")
|
|
45
|
+
return parser.parse_args()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _require(value, flag_name):
|
|
49
|
+
if value is None:
|
|
50
|
+
raise SystemExit(f"--yes was given but {flag_name} was not provided and is required.")
|
|
51
|
+
return value
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def main():
|
|
55
|
+
args = parse_args()
|
|
56
|
+
|
|
57
|
+
# --- data path ---
|
|
58
|
+
data_path = args.data
|
|
59
|
+
if data_path is None:
|
|
60
|
+
if args.yes:
|
|
61
|
+
_require(None, "--data")
|
|
62
|
+
data_path = interactive.prompt_for_path()
|
|
63
|
+
|
|
64
|
+
# --- sheet selection (Excel only) ---
|
|
65
|
+
sheet = args.sheet
|
|
66
|
+
loader = TabularLoader(data_path)
|
|
67
|
+
available_sheets = loader.list_sheets()
|
|
68
|
+
if available_sheets and sheet is None and not args.yes:
|
|
69
|
+
sheet = interactive.prompt_for_sheet(available_sheets)
|
|
70
|
+
if sheet is not None:
|
|
71
|
+
loader = TabularLoader(data_path, sheet=sheet)
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
df = loader.load()
|
|
75
|
+
except (CorrelationEngineError, FileNotFoundError) as e:
|
|
76
|
+
raise SystemExit(f"[tlf-correlation-engine] {e}")
|
|
77
|
+
|
|
78
|
+
# --- method ---
|
|
79
|
+
method = args.method
|
|
80
|
+
if method is None:
|
|
81
|
+
if args.yes:
|
|
82
|
+
method = "pearson"
|
|
83
|
+
else:
|
|
84
|
+
method = interactive.prompt_for_method()
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
engine = CorrelationEngine(df, method=method)
|
|
88
|
+
except CorrelationEngineError as e:
|
|
89
|
+
raise SystemExit(f"[tlf-correlation-engine] {e}")
|
|
90
|
+
|
|
91
|
+
# --- output type ---
|
|
92
|
+
output = args.output
|
|
93
|
+
if output is None:
|
|
94
|
+
if args.yes:
|
|
95
|
+
output = "report"
|
|
96
|
+
else:
|
|
97
|
+
output = interactive.prompt_for_output()
|
|
98
|
+
|
|
99
|
+
# --- report filters (only relevant for output == "report") ---
|
|
100
|
+
threshold = args.threshold
|
|
101
|
+
significant_only = args.significant_only
|
|
102
|
+
if output == "report" and not args.yes and threshold is None and significant_only is None:
|
|
103
|
+
threshold, significant_only = interactive.prompt_for_report_filters()
|
|
104
|
+
significant_only = bool(significant_only)
|
|
105
|
+
|
|
106
|
+
# --- compute result ---
|
|
107
|
+
if output == "matrix":
|
|
108
|
+
result = engine.matrix()
|
|
109
|
+
elif output == "pvalues":
|
|
110
|
+
result = engine.pvalue_matrix()
|
|
111
|
+
else:
|
|
112
|
+
result = engine.report(threshold=threshold, significant_only=significant_only, alpha=args.alpha)
|
|
113
|
+
|
|
114
|
+
print(result.to_string())
|
|
115
|
+
|
|
116
|
+
# --- export ---
|
|
117
|
+
export_fmt, export_path = args.export, args.export_path
|
|
118
|
+
if export_fmt is None and not args.yes:
|
|
119
|
+
export_fmt, export_path = interactive.prompt_for_export()
|
|
120
|
+
if export_fmt:
|
|
121
|
+
if export_path is None:
|
|
122
|
+
_require(None, "--export-path")
|
|
123
|
+
_export(result, export_fmt, export_path)
|
|
124
|
+
print(f"\nExported to {export_path}")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _export(result: pd.DataFrame, fmt: str, path: str):
|
|
128
|
+
if fmt == "csv":
|
|
129
|
+
result.to_csv(path)
|
|
130
|
+
elif fmt == "json":
|
|
131
|
+
result.to_json(path, orient="records" if "variable_1" in result.columns else "split", indent=2)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__":
|
|
135
|
+
main()
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Pairwise correlation statistics.
|
|
3
|
+
|
|
4
|
+
Each function takes two 1-D numeric sequences (typically pandas Series),
|
|
5
|
+
drops rows where either value is missing, and returns a dict with the
|
|
6
|
+
correlation coefficient, its p-value, and the number of paired
|
|
7
|
+
observations actually used (``n``).
|
|
8
|
+
|
|
9
|
+
Three methods are supported, matching the TLF-Data-Analysis roadmap:
|
|
10
|
+
|
|
11
|
+
- ``pearson`` — linear correlation (scipy.stats.pearsonr)
|
|
12
|
+
- ``spearman`` — rank/monotonic correlation (scipy.stats.spearmanr)
|
|
13
|
+
- ``kendall`` — rank concordance correlation (scipy.stats.kendalltau)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
from scipy import stats
|
|
18
|
+
|
|
19
|
+
from .errors import InvalidMethodError, InsufficientDataError
|
|
20
|
+
|
|
21
|
+
# Below this many paired observations, a correlation coefficient is not
|
|
22
|
+
# considered statistically meaningful.
|
|
23
|
+
MIN_OBSERVATIONS = 3
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _paired_dropna(a, b):
|
|
27
|
+
"""Align two series on a shared index and drop rows where either is null."""
|
|
28
|
+
a = pd.Series(a)
|
|
29
|
+
b = pd.Series(b)
|
|
30
|
+
paired = pd.concat([a, b], axis=1).dropna()
|
|
31
|
+
return paired.iloc[:, 0], paired.iloc[:, 1]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def pearson(a, b):
|
|
35
|
+
a, b = _paired_dropna(a, b)
|
|
36
|
+
if len(a) < MIN_OBSERVATIONS:
|
|
37
|
+
return {"method": "pearson", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
|
|
38
|
+
stat, p = stats.pearsonr(a, b)
|
|
39
|
+
return {"method": "pearson", "coefficient": stat, "p_value": p, "n": len(a)}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def spearman(a, b):
|
|
43
|
+
a, b = _paired_dropna(a, b)
|
|
44
|
+
if len(a) < MIN_OBSERVATIONS:
|
|
45
|
+
return {"method": "spearman", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
|
|
46
|
+
stat, p = stats.spearmanr(a, b)
|
|
47
|
+
return {"method": "spearman", "coefficient": stat, "p_value": p, "n": len(a)}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def kendall(a, b):
|
|
51
|
+
a, b = _paired_dropna(a, b)
|
|
52
|
+
if len(a) < MIN_OBSERVATIONS:
|
|
53
|
+
return {"method": "kendall", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
|
|
54
|
+
stat, p = stats.kendalltau(a, b)
|
|
55
|
+
return {"method": "kendall", "coefficient": stat, "p_value": p, "n": len(a)}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
METHODS = {
|
|
59
|
+
"pearson": pearson,
|
|
60
|
+
"spearman": spearman,
|
|
61
|
+
"kendall": kendall,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def pairwise_correlation(a, b, method="pearson"):
|
|
66
|
+
"""Compute a single pairwise correlation between two series.
|
|
67
|
+
|
|
68
|
+
Returns a dict: {"method", "coefficient", "p_value", "n"}.
|
|
69
|
+
"""
|
|
70
|
+
if method not in METHODS:
|
|
71
|
+
raise InvalidMethodError(
|
|
72
|
+
f"Unsupported method '{method}'. Choose from: {', '.join(METHODS)}."
|
|
73
|
+
)
|
|
74
|
+
return METHODS[method](a, b)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tlf-correlation-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Country-agnostic Pearson/Spearman/Kendall correlation analysis over pandas DataFrames — part of The Living Facts (TLF).
|
|
5
|
+
Author-email: Sanchita Karki <karkisanchu06@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ctpl-git/TLF-Data-Analysis
|
|
8
|
+
Project-URL: Repository, https://github.com/ctpl-git/TLF-Data-Analysis
|
|
9
|
+
Project-URL: Issues, https://github.com/ctpl-git/TLF-Data-Analysis/issues
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pandas>=1.5
|
|
22
|
+
Requires-Dist: scipy>=1.9
|
|
23
|
+
Requires-Dist: openpyxl>=3.1
|
|
24
|
+
Requires-Dist: questionary>=2.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# tlf-correlation-engine
|
|
30
|
+
|
|
31
|
+
Country-agnostic correlation analysis — part of **TLF** ("The Living Facts").
|
|
32
|
+
|
|
33
|
+
Computes Pearson, Spearman, or Kendall correlations across any pandas
|
|
34
|
+
DataFrame's numeric columns, along with p-values and observation counts,
|
|
35
|
+
and produces a sorted, filterable long-form report of the strongest and
|
|
36
|
+
most statistically significant relationships in a dataset.
|
|
37
|
+
|
|
38
|
+
Unlike `tlf-census-stats`, this package has no dependency on a specific
|
|
39
|
+
country schema — it works on any DataFrame with 2+ numeric columns.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install tlf-correlation-engine
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or from source, inside the `TLF-Data-Analysis` monorepo:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
cd tlf-correlation-engine
|
|
53
|
+
pip install -e ".[dev]"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Usage
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import pandas as pd
|
|
62
|
+
from tlf_correlation_engine import CorrelationEngine
|
|
63
|
+
|
|
64
|
+
df = pd.read_csv("census_data.csv")
|
|
65
|
+
|
|
66
|
+
engine = CorrelationEngine(df, method="pearson") # or "spearman" / "kendall"
|
|
67
|
+
|
|
68
|
+
# Coefficient matrix (like df.corr(), but validated numeric-only)
|
|
69
|
+
engine.matrix()
|
|
70
|
+
|
|
71
|
+
# P-value matrix, aligned with matrix()
|
|
72
|
+
engine.pvalue_matrix()
|
|
73
|
+
|
|
74
|
+
# One specific pair, with n and p-value
|
|
75
|
+
engine.pairwise("literacy_rate", "urban_population")
|
|
76
|
+
|
|
77
|
+
# Long-form report of all pairs, strongest relationship first
|
|
78
|
+
engine.report()
|
|
79
|
+
|
|
80
|
+
# Only strong (|r| >= 0.5) and statistically significant (p < 0.05) pairs
|
|
81
|
+
engine.report(threshold=0.5, significant_only=True)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Methods
|
|
85
|
+
|
|
86
|
+
| Method | Use for |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `pearson` | Linear relationships between continuous variables |
|
|
89
|
+
| `spearman` | Monotonic (rank-based) relationships, robust to outliers/non-linearity |
|
|
90
|
+
| `kendall` | Rank concordance, more robust on small samples or many tied ranks |
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## CLI
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
tlf-correlation-engine --data census.csv --method spearman --output report --export csv --export-path out.csv
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Run with no flags at all for a fully interactive walkthrough (file path → sheet selection → method → output type → report filters → export format):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
tlf-correlation-engine
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
For unattended/scripted runs, `--yes` disables all prompting and fails loudly (rather than silently guessing) if something required — like `--data` — is missing:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
tlf-correlation-engine --data census.csv --yes --method pearson --output matrix
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### CLI flags
|
|
113
|
+
|
|
114
|
+
| Flag | Description |
|
|
115
|
+
|---|---|
|
|
116
|
+
| `--data` | Path to a CSV, Excel, or JSON file |
|
|
117
|
+
| `--sheet` | Excel sheet name (default: first sheet) |
|
|
118
|
+
| `--method` | `pearson` \| `spearman` \| `kendall` |
|
|
119
|
+
| `--output` | `report` \| `matrix` \| `pvalues` |
|
|
120
|
+
| `--threshold` | Report only: minimum \|coefficient\| to include |
|
|
121
|
+
| `--significant-only` | Report only: only include pairs with p < `--alpha` |
|
|
122
|
+
| `--alpha` | Significance threshold for `--significant-only` (default 0.05) |
|
|
123
|
+
| `--export` | `csv` \| `json` |
|
|
124
|
+
| `--export-path` | Export file path |
|
|
125
|
+
| `--yes` | Non-interactive mode: never prompt, error on missing required values |
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Errors
|
|
130
|
+
|
|
131
|
+
- `InvalidMethodError` — unsupported `method` value
|
|
132
|
+
- `InsufficientDataError` — fewer than 2 numeric columns in the DataFrame
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## License
|
|
137
|
+
|
|
138
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
tlf_correlation_engine/__init__.py,sha256=JGh4txQGh0aoouzHbFsToWgf31JB-r3lHMWIIkQ3Yso,867
|
|
2
|
+
tlf_correlation_engine/engine.py,sha256=35dgbl_TbwIcUGjQO79A4v15oamm_iPIQTX9a98JZKU,6052
|
|
3
|
+
tlf_correlation_engine/errors.py,sha256=B-jjSDhLZPBuqDqgOdGbMjf-OEAtfRc22GLpHOViA8E,443
|
|
4
|
+
tlf_correlation_engine/interactive.py,sha256=u4SR6svEF0LIyHA6CIHpuJz6UtP-tbigMu0vdtHXHEY,3068
|
|
5
|
+
tlf_correlation_engine/loader.py,sha256=VPB8xyO60k-raSuCec-UXeW0bh_aaapUS9NzktHCWX0,3337
|
|
6
|
+
tlf_correlation_engine/main.py,sha256=reRuTzob5Fjxu-UU_y_3M4yYXK1RGRQu-X-wQTIF8dE,5136
|
|
7
|
+
tlf_correlation_engine/methods.py,sha256=sOi22k5-gYUc4-QOYH4OTsTVF_H7Wj8dK0kf8axoJ80,2441
|
|
8
|
+
tlf_correlation_engine-0.1.0.dist-info/licenses/LICENSE,sha256=Oi5Nz1L9eFpL3SUampJQJMvTKHVvGdPF-bEaHxFeqlk,1070
|
|
9
|
+
tlf_correlation_engine-0.1.0.dist-info/METADATA,sha256=HOdRIUAayptUhQTZ6Th8VcAuPuyB6nyzAZ5p32WJODA,4402
|
|
10
|
+
tlf_correlation_engine-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
11
|
+
tlf_correlation_engine-0.1.0.dist-info/entry_points.txt,sha256=ROELEpgkAf7EWXy0zw21t-IAu9VglpmvlPb_oBmXkMM,76
|
|
12
|
+
tlf_correlation_engine-0.1.0.dist-info/top_level.txt,sha256=e_PNyTDcSFJqfdAfgcquHUpClglpZpKM246AHqPsyeo,23
|
|
13
|
+
tlf_correlation_engine-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sanchita Karki
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tlf_correlation_engine
|