tlf-correlation-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,28 @@
1
+ """
2
+ tlf-correlation-engine
3
+ -----------------------
4
+ Country-agnostic correlation analysis (Pearson, Spearman, Kendall) over
5
+ any pandas DataFrame's numeric columns — coefficient matrices, p-values,
6
+ and a sortable/filterable long-form report of relationship strength.
7
+ Part of the TLF-Data-Analysis repo, part of the TLF ("The Living Facts")
8
+ initiative.
9
+ """
10
+
11
+ from .engine import CorrelationEngine, SUPPORTED_METHODS
12
+ from .methods import pairwise_correlation
13
+ from .loader import TabularLoader, UnsupportedFileError
14
+ from .errors import CorrelationEngineError, InvalidMethodError, InsufficientDataError
15
+
16
+ __all__ = [
17
+ "__version__",
18
+ "CorrelationEngine",
19
+ "SUPPORTED_METHODS",
20
+ "pairwise_correlation",
21
+ "TabularLoader",
22
+ "UnsupportedFileError",
23
+ "CorrelationEngineError",
24
+ "InvalidMethodError",
25
+ "InsufficientDataError",
26
+ ]
27
+
28
+ __version__ = "0.1.0"
@@ -0,0 +1,155 @@
1
+ """
2
+ Core CorrelationEngine — country-agnostic correlation analysis over any
3
+ pandas DataFrame's numeric columns.
4
+
5
+ Unlike a bare ``df.corr()`` call, this engine also reports statistical
6
+ significance (p-values) and the number of paired observations behind
7
+ each coefficient, and can produce a sorted, filterable long-form report
8
+ of the strongest / most significant relationships in a dataset.
9
+
10
+ matrix(), pvalue_matrix(), pairwise(), and report() are all backed by
11
+ the same MIN_OBSERVATIONS-aware pairwise_correlation() calls (see
12
+ methods.py), so they always agree with each other: a pair with too few
13
+ paired observations to be meaningful comes back as NaN everywhere, not
14
+ just in the p-value output.
15
+ """
16
+
17
+ import pandas as pd
18
+
19
+ from .errors import InvalidMethodError, InsufficientDataError
20
+ from .methods import METHODS, pairwise_correlation
21
+
22
+ SUPPORTED_METHODS = tuple(METHODS)
23
+
24
+
25
+ class CorrelationEngine:
26
+ """Compute correlations across a DataFrame's numeric columns.
27
+
28
+ Parameters
29
+ ----------
30
+ df : pandas.DataFrame
31
+ Any DataFrame. Only numeric columns are used; non-numeric
32
+ columns are ignored automatically.
33
+ method : str
34
+ One of "pearson" (default), "spearman", or "kendall".
35
+ """
36
+
37
+ def __init__(self, df: pd.DataFrame, method: str = "pearson"):
38
+ if method not in SUPPORTED_METHODS:
39
+ raise InvalidMethodError(
40
+ f"Unsupported method '{method}'. Choose from: {', '.join(SUPPORTED_METHODS)}."
41
+ )
42
+
43
+ self.method = method
44
+ self.df = df
45
+ self._numeric_df = df.select_dtypes(include="number")
46
+ self._grid_cache = None # set on first matrix()/pvalue_matrix() call
47
+
48
+ if self._numeric_df.shape[1] < 2:
49
+ raise InsufficientDataError(
50
+ "Need at least 2 numeric columns to compute correlations; "
51
+ f"found {self._numeric_df.shape[1]}."
52
+ )
53
+
54
+ @property
55
+ def columns(self):
56
+ """Numeric columns considered by this engine."""
57
+ return list(self._numeric_df.columns)
58
+
59
+ def _pairwise_grid(self):
60
+ """Computes the coefficient and p-value matrices in one pass, both
61
+ going through the same MIN_OBSERVATIONS-aware pairwise_correlation()
62
+ used by pairwise()/report() — so a pair with too few paired
63
+ observations is NaN everywhere, not just in the p-value matrix.
64
+ Cached per-instance since method/data don't change after __init__.
65
+ """
66
+ if self._grid_cache is not None:
67
+ return self._grid_cache
68
+
69
+ cols = self.columns
70
+ coef = pd.DataFrame(index=cols, columns=cols, dtype=float)
71
+ pval = pd.DataFrame(index=cols, columns=cols, dtype=float)
72
+
73
+ for i, col_a in enumerate(cols):
74
+ # A column is perfectly, trivially correlated with itself,
75
+ # regardless of n — this is a definitional identity, not a
76
+ # statistical claim, so MIN_OBSERVATIONS doesn't apply here.
77
+ coef.loc[col_a, col_a] = 1.0
78
+ pval.loc[col_a, col_a] = 0.0
79
+ for col_b in cols[i + 1:]:
80
+ result = pairwise_correlation(
81
+ self._numeric_df[col_a], self._numeric_df[col_b], method=self.method
82
+ )
83
+ coef.loc[col_a, col_b] = result["coefficient"]
84
+ coef.loc[col_b, col_a] = result["coefficient"]
85
+ pval.loc[col_a, col_b] = result["p_value"]
86
+ pval.loc[col_b, col_a] = result["p_value"]
87
+
88
+ self._grid_cache = (coef, pval)
89
+ return self._grid_cache
90
+
91
+ def matrix(self) -> pd.DataFrame:
92
+ """Correlation coefficient matrix. A pair with fewer than
93
+ MIN_OBSERVATIONS paired observations (see methods.py) is NaN,
94
+ matching pairwise()/report() — this is NOT the same as a bare
95
+ ``df.corr()`` call, which would compute a (statistically
96
+ meaningless) coefficient from as few as 2 points."""
97
+ coef, _ = self._pairwise_grid()
98
+ return coef
99
+
100
+ def pvalue_matrix(self) -> pd.DataFrame:
101
+ """P-value matrix aligned with ``matrix()``. Diagonal is 0.0 (a
102
+ column is perfectly, trivially correlated with itself)."""
103
+ _, pval = self._pairwise_grid()
104
+ return pval
105
+
106
+ def pairwise(self, col_a: str, col_b: str) -> dict:
107
+ """Correlation between two specific columns, with n and p-value."""
108
+ return pairwise_correlation(
109
+ self._numeric_df[col_a], self._numeric_df[col_b], method=self.method
110
+ )
111
+
112
+ def report(self, threshold: float = None, significant_only: bool = False, alpha: float = 0.05) -> pd.DataFrame:
113
+ """Long-form report: one row per unique column pair, sorted by
114
+ the strength of the relationship (descending |coefficient|).
115
+
116
+ Parameters
117
+ ----------
118
+ threshold : float, optional
119
+ Only include pairs where |coefficient| >= threshold.
120
+ significant_only : bool
121
+ Only include pairs where p_value < alpha.
122
+ alpha : float
123
+ Significance threshold used when significant_only=True.
124
+ """
125
+ cols = self.columns
126
+ rows = []
127
+
128
+ for i, col_a in enumerate(cols):
129
+ for col_b in cols[i + 1:]:
130
+ result = self.pairwise(col_a, col_b)
131
+ rows.append({
132
+ "variable_1": col_a,
133
+ "variable_2": col_b,
134
+ "method": result["method"],
135
+ "coefficient": result["coefficient"],
136
+ "p_value": result["p_value"],
137
+ "n": result["n"],
138
+ })
139
+
140
+ report_df = pd.DataFrame(rows)
141
+
142
+ if report_df.empty:
143
+ return report_df
144
+
145
+ if threshold is not None:
146
+ report_df = report_df[report_df["coefficient"].abs() >= threshold]
147
+
148
+ if significant_only:
149
+ report_df = report_df[report_df["p_value"] < alpha]
150
+
151
+ report_df = report_df.reindex(
152
+ report_df["coefficient"].abs().sort_values(ascending=False).index
153
+ ).reset_index(drop=True)
154
+
155
+ return report_df
@@ -0,0 +1,14 @@
1
+ """Exceptions raised by tlf_correlation_engine."""
2
+
3
+
4
+ class CorrelationEngineError(Exception):
5
+ """Base exception for tlf_correlation_engine."""
6
+
7
+
8
+ class InvalidMethodError(CorrelationEngineError):
9
+ """Raised when an unsupported correlation method is requested."""
10
+
11
+
12
+ class InsufficientDataError(CorrelationEngineError):
13
+ """Raised when a DataFrame doesn't have enough numeric columns or rows
14
+ to compute a meaningful correlation."""
@@ -0,0 +1,83 @@
1
+ """
2
+ interactive.py — prompts
3
+ Terminal prompts for any run configuration the user didn't supply via
4
+ CLI flags. Flags always win; a prompt only fires for whatever's still
5
+ missing after argument parsing. In --yes (non-interactive) mode, none
6
+ of these are called at all — main.py raises a clear error instead if
7
+ something required is still missing.
8
+
9
+ Uses questionary for arrow-key menus and multi-select checklists.
10
+ """
11
+
12
+ import questionary
13
+
14
+ from .engine import SUPPORTED_METHODS
15
+
16
+
17
+ def prompt_for_path() -> str:
18
+ return questionary.path("Path to your data file (CSV, Excel, or JSON):").ask()
19
+
20
+
21
+ def prompt_for_sheet(sheet_names: list):
22
+ if not sheet_names or len(sheet_names) == 1:
23
+ return sheet_names[0] if sheet_names else None
24
+ return questionary.select("Which sheet should be read?", choices=sheet_names).ask()
25
+
26
+
27
+ def prompt_for_method() -> str:
28
+ return questionary.select(
29
+ "Which correlation method?",
30
+ choices=[
31
+ questionary.Choice("Pearson (linear relationships)", value="pearson"),
32
+ questionary.Choice("Spearman (monotonic, rank-based, robust to outliers)", value="spearman"),
33
+ questionary.Choice("Kendall (rank concordance, good for small samples)", value="kendall"),
34
+ ],
35
+ ).ask()
36
+
37
+
38
+ def prompt_for_output() -> str:
39
+ return questionary.select(
40
+ "What should be shown?",
41
+ choices=[
42
+ questionary.Choice("Long-form report (sorted by relationship strength)", value="report"),
43
+ questionary.Choice("Correlation coefficient matrix", value="matrix"),
44
+ questionary.Choice("P-value matrix", value="pvalues"),
45
+ ],
46
+ ).ask()
47
+
48
+
49
+ def prompt_for_report_filters() -> tuple:
50
+ """Only asked when output == 'report'. Returns (threshold, significant_only)."""
51
+ significant_only = questionary.confirm(
52
+ "Only show statistically significant pairs (p < 0.05)?", default=False
53
+ ).ask()
54
+
55
+ use_threshold = questionary.confirm(
56
+ "Filter by minimum correlation strength?", default=False
57
+ ).ask()
58
+ threshold = None
59
+ if use_threshold:
60
+ raw = questionary.text("Minimum |coefficient| (0.0–1.0):", default="0.5").ask()
61
+ threshold = float(raw)
62
+
63
+ return threshold, significant_only
64
+
65
+
66
+ def prompt_for_export() -> tuple:
67
+ # NOTE: questionary.Choice(value=None) is treated as "no value given"
68
+ # and defaults to the choice's *title* text, not to a real None. So
69
+ # "no export" needs its own distinct sentinel ("none") rather than
70
+ # value=None, or selecting it would be silently mistaken for a
71
+ # (nonsense) export format and still try to prompt for a path.
72
+ fmt = questionary.select(
73
+ "Export the result to a file?",
74
+ choices=[
75
+ questionary.Choice("No, just print to terminal", value="none"),
76
+ questionary.Choice("CSV", value="csv"),
77
+ questionary.Choice("JSON", value="json"),
78
+ ],
79
+ ).ask()
80
+ if fmt in (None, "none"):
81
+ return None, None
82
+ path = questionary.text(f"Export path (e.g. output.{fmt}):").ask()
83
+ return fmt, path
@@ -0,0 +1,95 @@
1
+ """
2
+ loader.py — TabularLoader
3
+ Reads a CSV, Excel, or JSON file into a plain pandas DataFrame with no
4
+ schema assumptions: no required columns, no renaming. Column headers
5
+ are used exactly as given in the source file — CorrelationEngine picks
6
+ out the numeric ones itself.
7
+
8
+ This mirrors tlf-statistical-summary's TabularLoader (same supported
9
+ formats, same PDF rejection message) since both packages are meant to
10
+ sit downstream of already-cleaned data, not parse raw PDFs themselves.
11
+ """
12
+
13
+ import json
14
+ from pathlib import Path
15
+
16
+ import pandas as pd
17
+
18
+ from .errors import CorrelationEngineError
19
+
20
+
21
+ class UnsupportedFileError(CorrelationEngineError):
22
+ """Raised when the input file's type can't be read at all (e.g. a
23
+ raw PDF, or an extension this package doesn't recognize)."""
24
+
25
+
26
+ class TabularLoader:
27
+ """
28
+ Loads tabular data from a CSV, Excel, or JSON file, as-is.
29
+
30
+ Usage:
31
+ loader = TabularLoader("census_data.csv")
32
+ df = loader.load()
33
+
34
+ loader = TabularLoader("data.xlsx", sheet="District Summary")
35
+ df = loader.load()
36
+ """
37
+
38
+ SUPPORTED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".json"}
39
+
40
+ def __init__(self, filepath: str, sheet=None):
41
+ self.filepath = Path(filepath)
42
+ # Excel only: None = first/only sheet; a name or index restricts to one.
43
+ self.sheet = sheet
44
+
45
+ def load(self) -> pd.DataFrame:
46
+ if not self.filepath.exists():
47
+ raise FileNotFoundError(f"Data file not found: {self.filepath}")
48
+ return self._read_file()
49
+
50
+ def _read_file(self) -> pd.DataFrame:
51
+ ext = self.filepath.suffix.lower()
52
+
53
+ if ext == ".csv":
54
+ return pd.read_csv(self.filepath)
55
+
56
+ if ext in (".xlsx", ".xls"):
57
+ sheet = self.sheet if self.sheet is not None else 0
58
+ return pd.read_excel(self.filepath, sheet_name=sheet)
59
+
60
+ if ext == ".json":
61
+ return self._read_json()
62
+
63
+ if ext == ".pdf":
64
+ raise UnsupportedFileError(
65
+ f"Cannot read '{self.filepath.name}' directly: this package does not "
66
+ f"parse PDFs. Convert the PDF's tables into a CSV/Excel/JSON file "
67
+ f"first (e.g. with tlf-data-cleaning), then load that output here instead."
68
+ )
69
+
70
+ raise UnsupportedFileError(
71
+ f"Unsupported file type '{ext}' for '{self.filepath.name}'. "
72
+ f"This package reads: {sorted(self.SUPPORTED_EXTENSIONS)}."
73
+ )
74
+
75
+ def _read_json(self) -> pd.DataFrame:
76
+ with open(self.filepath, "r", encoding="utf-8") as f:
77
+ data = json.load(f)
78
+
79
+ if isinstance(data, list):
80
+ return pd.DataFrame(data)
81
+ if isinstance(data, dict):
82
+ return pd.DataFrame([data])
83
+ raise UnsupportedFileError(
84
+ f"'{self.filepath.name}' is valid JSON but not in a shape this package "
85
+ f"can read: expected a list of row-objects or a single flat object, "
86
+ f"got {type(data).__name__}."
87
+ )
88
+
89
+ def list_sheets(self) -> list:
90
+ """Returns the sheet names in an Excel workbook, for building an
91
+ interactive sheet-selection prompt. Empty list for non-Excel files."""
92
+ ext = self.filepath.suffix.lower()
93
+ if ext not in (".xlsx", ".xls"):
94
+ return []
95
+ return pd.ExcelFile(self.filepath).sheet_names
@@ -0,0 +1,135 @@
1
+ """
2
+ main.py — tlf-correlation-engine CLI
3
+ Country-agnostic Pearson/Spearman/Kendall correlation analysis for any
4
+ tabular file.
5
+
6
+ Usage:
7
+ python -m tlf_correlation_engine.main --data census.csv
8
+ python -m tlf_correlation_engine.main --data census.csv --method spearman --output matrix
9
+ python -m tlf_correlation_engine.main --data census.csv --yes --output report --export csv --export-path out.csv
10
+
11
+ Flags are read first; anything not supplied is filled in via
12
+ interactive terminal prompts, UNLESS --yes is passed, in which case
13
+ any required-but-missing value raises a clear error instead of
14
+ prompting (so this can run unattended, e.g. from a script or cron job,
15
+ without hanging on input()).
16
+ """
17
+
18
+ import argparse
19
+ import sys
20
+
21
+ import pandas as pd
22
+
23
+ from .loader import TabularLoader
24
+ from .engine import CorrelationEngine, SUPPORTED_METHODS
25
+ from .errors import CorrelationEngineError
26
+ from . import interactive
27
+
28
+
29
+ def parse_args():
30
+ parser = argparse.ArgumentParser(description="tlf-correlation-engine: Pearson/Spearman/Kendall correlation analysis")
31
+ parser.add_argument("--data", default=None, help="Path to a CSV, Excel, or JSON file.")
32
+ parser.add_argument("--sheet", default=None, help="Excel sheet name to read. Default: first sheet.")
33
+ parser.add_argument("--method", default=None, choices=SUPPORTED_METHODS, help="Correlation method.")
34
+ parser.add_argument("--output", default=None, choices=["report", "matrix", "pvalues"],
35
+ help="What to show: long-form report, coefficient matrix, or p-value matrix.")
36
+ parser.add_argument("--threshold", type=float, default=None,
37
+ help="Report only: minimum |coefficient| to include.")
38
+ parser.add_argument("--significant-only", dest="significant_only", action="store_true", default=None,
39
+ help="Report only: only include pairs with p < --alpha.")
40
+ parser.add_argument("--alpha", type=float, default=0.05, help="Significance threshold for --significant-only.")
41
+ parser.add_argument("--export", default=None, choices=["csv", "json"], help="Export format.")
42
+ parser.add_argument("--export-path", default=None, help="Export file path.")
43
+ parser.add_argument("--yes", action="store_true",
44
+ help="Non-interactive mode: never prompt, error on missing required values.")
45
+ return parser.parse_args()
46
+
47
+
48
+ def _require(value, flag_name):
49
+ if value is None:
50
+ raise SystemExit(f"--yes was given but {flag_name} was not provided and is required.")
51
+ return value
52
+
53
+
54
+ def main():
55
+ args = parse_args()
56
+
57
+ # --- data path ---
58
+ data_path = args.data
59
+ if data_path is None:
60
+ if args.yes:
61
+ _require(None, "--data")
62
+ data_path = interactive.prompt_for_path()
63
+
64
+ # --- sheet selection (Excel only) ---
65
+ sheet = args.sheet
66
+ loader = TabularLoader(data_path)
67
+ available_sheets = loader.list_sheets()
68
+ if available_sheets and sheet is None and not args.yes:
69
+ sheet = interactive.prompt_for_sheet(available_sheets)
70
+ if sheet is not None:
71
+ loader = TabularLoader(data_path, sheet=sheet)
72
+
73
+ try:
74
+ df = loader.load()
75
+ except (CorrelationEngineError, FileNotFoundError) as e:
76
+ raise SystemExit(f"[tlf-correlation-engine] {e}")
77
+
78
+ # --- method ---
79
+ method = args.method
80
+ if method is None:
81
+ if args.yes:
82
+ method = "pearson"
83
+ else:
84
+ method = interactive.prompt_for_method()
85
+
86
+ try:
87
+ engine = CorrelationEngine(df, method=method)
88
+ except CorrelationEngineError as e:
89
+ raise SystemExit(f"[tlf-correlation-engine] {e}")
90
+
91
+ # --- output type ---
92
+ output = args.output
93
+ if output is None:
94
+ if args.yes:
95
+ output = "report"
96
+ else:
97
+ output = interactive.prompt_for_output()
98
+
99
+ # --- report filters (only relevant for output == "report") ---
100
+ threshold = args.threshold
101
+ significant_only = args.significant_only
102
+ if output == "report" and not args.yes and threshold is None and significant_only is None:
103
+ threshold, significant_only = interactive.prompt_for_report_filters()
104
+ significant_only = bool(significant_only)
105
+
106
+ # --- compute result ---
107
+ if output == "matrix":
108
+ result = engine.matrix()
109
+ elif output == "pvalues":
110
+ result = engine.pvalue_matrix()
111
+ else:
112
+ result = engine.report(threshold=threshold, significant_only=significant_only, alpha=args.alpha)
113
+
114
+ print(result.to_string())
115
+
116
+ # --- export ---
117
+ export_fmt, export_path = args.export, args.export_path
118
+ if export_fmt is None and not args.yes:
119
+ export_fmt, export_path = interactive.prompt_for_export()
120
+ if export_fmt:
121
+ if export_path is None:
122
+ _require(None, "--export-path")
123
+ _export(result, export_fmt, export_path)
124
+ print(f"\nExported to {export_path}")
125
+
126
+
127
+ def _export(result: pd.DataFrame, fmt: str, path: str):
128
+ if fmt == "csv":
129
+ result.to_csv(path)
130
+ elif fmt == "json":
131
+ result.to_json(path, orient="records" if "variable_1" in result.columns else "split", indent=2)
132
+
133
+
134
+ if __name__ == "__main__":
135
+ main()
@@ -0,0 +1,74 @@
1
+ """
2
+ Pairwise correlation statistics.
3
+
4
+ Each function takes two 1-D numeric sequences (typically pandas Series),
5
+ drops rows where either value is missing, and returns a dict with the
6
+ correlation coefficient, its p-value, and the number of paired
7
+ observations actually used (``n``).
8
+
9
+ Three methods are supported, matching the TLF-Data-Analysis roadmap:
10
+
11
+ - ``pearson`` — linear correlation (scipy.stats.pearsonr)
12
+ - ``spearman`` — rank/monotonic correlation (scipy.stats.spearmanr)
13
+ - ``kendall`` — rank concordance correlation (scipy.stats.kendalltau)
14
+ """
15
+
16
+ import pandas as pd
17
+ from scipy import stats
18
+
19
+ from .errors import InvalidMethodError, InsufficientDataError
20
+
21
+ # Below this many paired observations, a correlation coefficient is not
22
+ # considered statistically meaningful.
23
+ MIN_OBSERVATIONS = 3
24
+
25
+
26
+ def _paired_dropna(a, b):
27
+ """Align two series on a shared index and drop rows where either is null."""
28
+ a = pd.Series(a)
29
+ b = pd.Series(b)
30
+ paired = pd.concat([a, b], axis=1).dropna()
31
+ return paired.iloc[:, 0], paired.iloc[:, 1]
32
+
33
+
34
+ def pearson(a, b):
35
+ a, b = _paired_dropna(a, b)
36
+ if len(a) < MIN_OBSERVATIONS:
37
+ return {"method": "pearson", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
38
+ stat, p = stats.pearsonr(a, b)
39
+ return {"method": "pearson", "coefficient": stat, "p_value": p, "n": len(a)}
40
+
41
+
42
+ def spearman(a, b):
43
+ a, b = _paired_dropna(a, b)
44
+ if len(a) < MIN_OBSERVATIONS:
45
+ return {"method": "spearman", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
46
+ stat, p = stats.spearmanr(a, b)
47
+ return {"method": "spearman", "coefficient": stat, "p_value": p, "n": len(a)}
48
+
49
+
50
+ def kendall(a, b):
51
+ a, b = _paired_dropna(a, b)
52
+ if len(a) < MIN_OBSERVATIONS:
53
+ return {"method": "kendall", "coefficient": float("nan"), "p_value": float("nan"), "n": len(a)}
54
+ stat, p = stats.kendalltau(a, b)
55
+ return {"method": "kendall", "coefficient": stat, "p_value": p, "n": len(a)}
56
+
57
+
58
+ METHODS = {
59
+ "pearson": pearson,
60
+ "spearman": spearman,
61
+ "kendall": kendall,
62
+ }
63
+
64
+
65
+ def pairwise_correlation(a, b, method="pearson"):
66
+ """Compute a single pairwise correlation between two series.
67
+
68
+ Returns a dict: {"method", "coefficient", "p_value", "n"}.
69
+ """
70
+ if method not in METHODS:
71
+ raise InvalidMethodError(
72
+ f"Unsupported method '{method}'. Choose from: {', '.join(METHODS)}."
73
+ )
74
+ return METHODS[method](a, b)
@@ -0,0 +1,138 @@
1
+ Metadata-Version: 2.4
2
+ Name: tlf-correlation-engine
3
+ Version: 0.1.0
4
+ Summary: Country-agnostic Pearson/Spearman/Kendall correlation analysis over pandas DataFrames — part of The Living Facts (TLF).
5
+ Author-email: Sanchita Karki <karkisanchu06@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/ctpl-git/TLF-Data-Analysis
8
+ Project-URL: Repository, https://github.com/ctpl-git/TLF-Data-Analysis
9
+ Project-URL: Issues, https://github.com/ctpl-git/TLF-Data-Analysis/issues
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.9
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
18
+ Requires-Python: >=3.9
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: pandas>=1.5
22
+ Requires-Dist: scipy>=1.9
23
+ Requires-Dist: openpyxl>=3.1
24
+ Requires-Dist: questionary>=2.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=7.0; extra == "dev"
27
+ Dynamic: license-file
28
+
29
+ # tlf-correlation-engine
30
+
31
+ Country-agnostic correlation analysis — part of **TLF** ("The Living Facts").
32
+
33
+ Computes Pearson, Spearman, or Kendall correlations across any pandas
34
+ DataFrame's numeric columns, along with p-values and observation counts,
35
+ and produces a sorted, filterable long-form report of the strongest and
36
+ most statistically significant relationships in a dataset.
37
+
38
+ Unlike `tlf-census-stats`, this package has no dependency on a specific
39
+ country schema — it works on any DataFrame with 2+ numeric columns.
40
+
41
+ ---
42
+
43
+ ## Install
44
+
45
+ ```bash
46
+ pip install tlf-correlation-engine
47
+ ```
48
+
49
+ Or from source, inside the `TLF-Data-Analysis` monorepo:
50
+
51
+ ```bash
52
+ cd tlf-correlation-engine
53
+ pip install -e ".[dev]"
54
+ ```
55
+
56
+ ---
57
+
58
+ ## Usage
59
+
60
+ ```python
61
+ import pandas as pd
62
+ from tlf_correlation_engine import CorrelationEngine
63
+
64
+ df = pd.read_csv("census_data.csv")
65
+
66
+ engine = CorrelationEngine(df, method="pearson") # or "spearman" / "kendall"
67
+
68
+ # Coefficient matrix (like df.corr(), but validated numeric-only)
69
+ engine.matrix()
70
+
71
+ # P-value matrix, aligned with matrix()
72
+ engine.pvalue_matrix()
73
+
74
+ # One specific pair, with n and p-value
75
+ engine.pairwise("literacy_rate", "urban_population")
76
+
77
+ # Long-form report of all pairs, strongest relationship first
78
+ engine.report()
79
+
80
+ # Only strong (|r| >= 0.5) and statistically significant (p < 0.05) pairs
81
+ engine.report(threshold=0.5, significant_only=True)
82
+ ```
83
+
84
+ ### Methods
85
+
86
+ | Method | Use for |
87
+ |---|---|
88
+ | `pearson` | Linear relationships between continuous variables |
89
+ | `spearman` | Monotonic (rank-based) relationships, robust to outliers/non-linearity |
90
+ | `kendall` | Rank concordance, more robust on small samples or many tied ranks |
91
+
92
+ ---
93
+
94
+ ## CLI
95
+
96
+ ```bash
97
+ tlf-correlation-engine --data census.csv --method spearman --output report --export csv --export-path out.csv
98
+ ```
99
+
100
+ Run with no flags at all for a fully interactive walkthrough (file path → sheet selection → method → output type → report filters → export format):
101
+
102
+ ```bash
103
+ tlf-correlation-engine
104
+ ```
105
+
106
+ For unattended/scripted runs, `--yes` disables all prompting and fails loudly (rather than silently guessing) if something required — like `--data` — is missing:
107
+
108
+ ```bash
109
+ tlf-correlation-engine --data census.csv --yes --method pearson --output matrix
110
+ ```
111
+
112
+ ### CLI flags
113
+
114
+ | Flag | Description |
115
+ |---|---|
116
+ | `--data` | Path to a CSV, Excel, or JSON file |
117
+ | `--sheet` | Excel sheet name (default: first sheet) |
118
+ | `--method` | `pearson` \| `spearman` \| `kendall` |
119
+ | `--output` | `report` \| `matrix` \| `pvalues` |
120
+ | `--threshold` | Report only: minimum \|coefficient\| to include |
121
+ | `--significant-only` | Report only: only include pairs with p < `--alpha` |
122
+ | `--alpha` | Significance threshold for `--significant-only` (default 0.05) |
123
+ | `--export` | `csv` \| `json` |
124
+ | `--export-path` | Export file path |
125
+ | `--yes` | Non-interactive mode: never prompt, error on missing required values |
126
+
127
+ ---
128
+
129
+ ## Errors
130
+
131
+ - `InvalidMethodError` — unsupported `method` value
132
+ - `InsufficientDataError` — fewer than 2 numeric columns in the DataFrame
133
+
134
+ ---
135
+
136
+ ## License
137
+
138
+ MIT — see `LICENSE`.
@@ -0,0 +1,13 @@
1
+ tlf_correlation_engine/__init__.py,sha256=JGh4txQGh0aoouzHbFsToWgf31JB-r3lHMWIIkQ3Yso,867
2
+ tlf_correlation_engine/engine.py,sha256=35dgbl_TbwIcUGjQO79A4v15oamm_iPIQTX9a98JZKU,6052
3
+ tlf_correlation_engine/errors.py,sha256=B-jjSDhLZPBuqDqgOdGbMjf-OEAtfRc22GLpHOViA8E,443
4
+ tlf_correlation_engine/interactive.py,sha256=u4SR6svEF0LIyHA6CIHpuJz6UtP-tbigMu0vdtHXHEY,3068
5
+ tlf_correlation_engine/loader.py,sha256=VPB8xyO60k-raSuCec-UXeW0bh_aaapUS9NzktHCWX0,3337
6
+ tlf_correlation_engine/main.py,sha256=reRuTzob5Fjxu-UU_y_3M4yYXK1RGRQu-X-wQTIF8dE,5136
7
+ tlf_correlation_engine/methods.py,sha256=sOi22k5-gYUc4-QOYH4OTsTVF_H7Wj8dK0kf8axoJ80,2441
8
+ tlf_correlation_engine-0.1.0.dist-info/licenses/LICENSE,sha256=Oi5Nz1L9eFpL3SUampJQJMvTKHVvGdPF-bEaHxFeqlk,1070
9
+ tlf_correlation_engine-0.1.0.dist-info/METADATA,sha256=HOdRIUAayptUhQTZ6Th8VcAuPuyB6nyzAZ5p32WJODA,4402
10
+ tlf_correlation_engine-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
11
+ tlf_correlation_engine-0.1.0.dist-info/entry_points.txt,sha256=ROELEpgkAf7EWXy0zw21t-IAu9VglpmvlPb_oBmXkMM,76
12
+ tlf_correlation_engine-0.1.0.dist-info/top_level.txt,sha256=e_PNyTDcSFJqfdAfgcquHUpClglpZpKM246AHqPsyeo,23
13
+ tlf_correlation_engine-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ tlf-correlation-engine = tlf_correlation_engine.main:main
@@ -0,0 +1,9 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sanchita Karki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
6
+
7
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1 @@
1
+ tlf_correlation_engine