tlf-correlation-engine 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tlf_correlation_engine-0.1.0/LICENSE +9 -0
- tlf_correlation_engine-0.1.0/PKG-INFO +138 -0
- tlf_correlation_engine-0.1.0/README.md +110 -0
- tlf_correlation_engine-0.1.0/pyproject.toml +45 -0
- tlf_correlation_engine-0.1.0/setup.cfg +4 -0
- tlf_correlation_engine-0.1.0/tests/test_engine.py +183 -0
- tlf_correlation_engine-0.1.0/tests/test_interactive.py +118 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/__init__.py +28 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/engine.py +155 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/errors.py +14 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/interactive.py +83 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/loader.py +95 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/main.py +135 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine/methods.py +74 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/PKG-INFO +138 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/SOURCES.txt +18 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/dependency_links.txt +1 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/entry_points.txt +2 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/requires.txt +7 -0
- tlf_correlation_engine-0.1.0/tlf_correlation_engine.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sanchita Karki
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tlf-correlation-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Country-agnostic Pearson/Spearman/Kendall correlation analysis over pandas DataFrames — part of The Living Facts (TLF).
|
|
5
|
+
Author-email: Sanchita Karki <karkisanchu06@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ctpl-git/TLF-Data-Analysis
|
|
8
|
+
Project-URL: Repository, https://github.com/ctpl-git/TLF-Data-Analysis
|
|
9
|
+
Project-URL: Issues, https://github.com/ctpl-git/TLF-Data-Analysis/issues
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pandas>=1.5
|
|
22
|
+
Requires-Dist: scipy>=1.9
|
|
23
|
+
Requires-Dist: openpyxl>=3.1
|
|
24
|
+
Requires-Dist: questionary>=2.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# tlf-correlation-engine
|
|
30
|
+
|
|
31
|
+
Country-agnostic correlation analysis — part of **TLF** ("The Living Facts").
|
|
32
|
+
|
|
33
|
+
Computes Pearson, Spearman, or Kendall correlations across any pandas
|
|
34
|
+
DataFrame's numeric columns, along with p-values and observation counts,
|
|
35
|
+
and produces a sorted, filterable long-form report of the strongest and
|
|
36
|
+
most statistically significant relationships in a dataset.
|
|
37
|
+
|
|
38
|
+
Unlike `tlf-census-stats`, this package has no dependency on a specific
|
|
39
|
+
country schema — it works on any DataFrame with 2+ numeric columns.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install tlf-correlation-engine
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or from source, inside the `TLF-Data-Analysis` monorepo:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
cd tlf-correlation-engine
|
|
53
|
+
pip install -e ".[dev]"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Usage
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import pandas as pd
|
|
62
|
+
from tlf_correlation_engine import CorrelationEngine
|
|
63
|
+
|
|
64
|
+
df = pd.read_csv("census_data.csv")
|
|
65
|
+
|
|
66
|
+
engine = CorrelationEngine(df, method="pearson") # or "spearman" / "kendall"
|
|
67
|
+
|
|
68
|
+
# Coefficient matrix (like df.corr(), but validated numeric-only)
|
|
69
|
+
engine.matrix()
|
|
70
|
+
|
|
71
|
+
# P-value matrix, aligned with matrix()
|
|
72
|
+
engine.pvalue_matrix()
|
|
73
|
+
|
|
74
|
+
# One specific pair, with n and p-value
|
|
75
|
+
engine.pairwise("literacy_rate", "urban_population")
|
|
76
|
+
|
|
77
|
+
# Long-form report of all pairs, strongest relationship first
|
|
78
|
+
engine.report()
|
|
79
|
+
|
|
80
|
+
# Only strong (|r| >= 0.5) and statistically significant (p < 0.05) pairs
|
|
81
|
+
engine.report(threshold=0.5, significant_only=True)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Methods
|
|
85
|
+
|
|
86
|
+
| Method | Use for |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `pearson` | Linear relationships between continuous variables |
|
|
89
|
+
| `spearman` | Monotonic (rank-based) relationships, robust to outliers/non-linearity |
|
|
90
|
+
| `kendall` | Rank concordance, more robust on small samples or many tied ranks |
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## CLI
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
tlf-correlation-engine --data census.csv --method spearman --output report --export csv --export-path out.csv
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Run with no flags at all for a fully interactive walkthrough (file path → sheet selection → method → output type → report filters → export format):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
tlf-correlation-engine
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
For unattended/scripted runs, `--yes` disables all prompting and fails loudly (rather than silently guessing) if something required — like `--data` — is missing:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
tlf-correlation-engine --data census.csv --yes --method pearson --output matrix
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### CLI flags
|
|
113
|
+
|
|
114
|
+
| Flag | Description |
|
|
115
|
+
|---|---|
|
|
116
|
+
| `--data` | Path to a CSV, Excel, or JSON file |
|
|
117
|
+
| `--sheet` | Excel sheet name (default: first sheet) |
|
|
118
|
+
| `--method` | `pearson` \| `spearman` \| `kendall` |
|
|
119
|
+
| `--output` | `report` \| `matrix` \| `pvalues` |
|
|
120
|
+
| `--threshold` | Report only: minimum \|coefficient\| to include |
|
|
121
|
+
| `--significant-only` | Report only: only include pairs with p < `--alpha` |
|
|
122
|
+
| `--alpha` | Significance threshold for `--significant-only` (default 0.05) |
|
|
123
|
+
| `--export` | `csv` \| `json` |
|
|
124
|
+
| `--export-path` | Export file path |
|
|
125
|
+
| `--yes` | Non-interactive mode: never prompt, error on missing required values |
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Errors
|
|
130
|
+
|
|
131
|
+
- `InvalidMethodError` — unsupported `method` value
|
|
132
|
+
- `InsufficientDataError` — fewer than 2 numeric columns in the DataFrame
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## License
|
|
137
|
+
|
|
138
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# tlf-correlation-engine
|
|
2
|
+
|
|
3
|
+
Country-agnostic correlation analysis — part of **TLF** ("The Living Facts").
|
|
4
|
+
|
|
5
|
+
Computes Pearson, Spearman, or Kendall correlations across any pandas
|
|
6
|
+
DataFrame's numeric columns, along with p-values and observation counts,
|
|
7
|
+
and produces a sorted, filterable long-form report of the strongest and
|
|
8
|
+
most statistically significant relationships in a dataset.
|
|
9
|
+
|
|
10
|
+
Unlike `tlf-census-stats`, this package has no dependency on a specific
|
|
11
|
+
country schema — it works on any DataFrame with 2+ numeric columns.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install tlf-correlation-engine
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Or from source, inside the `TLF-Data-Analysis` monorepo:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
cd tlf-correlation-engine
|
|
25
|
+
pip install -e ".[dev]"
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Usage
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
import pandas as pd
|
|
34
|
+
from tlf_correlation_engine import CorrelationEngine
|
|
35
|
+
|
|
36
|
+
df = pd.read_csv("census_data.csv")
|
|
37
|
+
|
|
38
|
+
engine = CorrelationEngine(df, method="pearson") # or "spearman" / "kendall"
|
|
39
|
+
|
|
40
|
+
# Coefficient matrix (like df.corr(), but validated numeric-only)
|
|
41
|
+
engine.matrix()
|
|
42
|
+
|
|
43
|
+
# P-value matrix, aligned with matrix()
|
|
44
|
+
engine.pvalue_matrix()
|
|
45
|
+
|
|
46
|
+
# One specific pair, with n and p-value
|
|
47
|
+
engine.pairwise("literacy_rate", "urban_population")
|
|
48
|
+
|
|
49
|
+
# Long-form report of all pairs, strongest relationship first
|
|
50
|
+
engine.report()
|
|
51
|
+
|
|
52
|
+
# Only strong (|r| >= 0.5) and statistically significant (p < 0.05) pairs
|
|
53
|
+
engine.report(threshold=0.5, significant_only=True)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Methods
|
|
57
|
+
|
|
58
|
+
| Method | Use for |
|
|
59
|
+
|---|---|
|
|
60
|
+
| `pearson` | Linear relationships between continuous variables |
|
|
61
|
+
| `spearman` | Monotonic (rank-based) relationships, robust to outliers/non-linearity |
|
|
62
|
+
| `kendall` | Rank concordance, more robust on small samples or many tied ranks |
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## CLI
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
tlf-correlation-engine --data census.csv --method spearman --output report --export csv --export-path out.csv
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Run with no flags at all for a fully interactive walkthrough (file path → sheet selection → method → output type → report filters → export format):
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
tlf-correlation-engine
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
For unattended/scripted runs, `--yes` disables all prompting and fails loudly (rather than silently guessing) if something required — like `--data` — is missing:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
tlf-correlation-engine --data census.csv --yes --method pearson --output matrix
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### CLI flags
|
|
85
|
+
|
|
86
|
+
| Flag | Description |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `--data` | Path to a CSV, Excel, or JSON file |
|
|
89
|
+
| `--sheet` | Excel sheet name (default: first sheet) |
|
|
90
|
+
| `--method` | `pearson` \| `spearman` \| `kendall` |
|
|
91
|
+
| `--output` | `report` \| `matrix` \| `pvalues` |
|
|
92
|
+
| `--threshold` | Report only: minimum \|coefficient\| to include |
|
|
93
|
+
| `--significant-only` | Report only: only include pairs with p < `--alpha` |
|
|
94
|
+
| `--alpha` | Significance threshold for `--significant-only` (default 0.05) |
|
|
95
|
+
| `--export` | `csv` \| `json` |
|
|
96
|
+
| `--export-path` | Export file path |
|
|
97
|
+
| `--yes` | Non-interactive mode: never prompt, error on missing required values |
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Errors
|
|
102
|
+
|
|
103
|
+
- `InvalidMethodError` — unsupported `method` value
|
|
104
|
+
- `InsufficientDataError` — fewer than 2 numeric columns in the DataFrame
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## License
|
|
109
|
+
|
|
110
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tlf-correlation-engine"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Country-agnostic Pearson/Spearman/Kendall correlation analysis over pandas DataFrames — part of The Living Facts (TLF)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Sanchita Karki", email = "karkisanchu06@gmail.com" },
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.9"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
dependencies = [
|
|
27
|
+
"pandas>=1.5",
|
|
28
|
+
"scipy>=1.9",
|
|
29
|
+
"openpyxl>=3.1",
|
|
30
|
+
"questionary>=2.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
dev = ["pytest>=7.0"]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/ctpl-git/TLF-Data-Analysis"
|
|
38
|
+
Repository = "https://github.com/ctpl-git/TLF-Data-Analysis"
|
|
39
|
+
Issues = "https://github.com/ctpl-git/TLF-Data-Analysis/issues"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
tlf-correlation-engine = "tlf_correlation_engine.main:main"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools]
|
|
45
|
+
packages = ["tlf_correlation_engine"]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from tlf_correlation_engine import (
|
|
7
|
+
CorrelationEngine,
|
|
8
|
+
pairwise_correlation,
|
|
9
|
+
InvalidMethodError,
|
|
10
|
+
InsufficientDataError,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.fixture
|
|
17
|
+
def census_df():
|
|
18
|
+
return pd.read_csv(os.path.join(FIXTURES_DIR, "nepal_census_2021.csv"))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.fixture
|
|
22
|
+
def perfectly_correlated_df():
|
|
23
|
+
return pd.DataFrame({
|
|
24
|
+
"a": [1, 2, 3, 4, 5],
|
|
25
|
+
"b": [2, 4, 6, 8, 10], # perfectly, positively correlated with a
|
|
26
|
+
"c": [10, 8, 6, 4, 2], # perfectly, negatively correlated with a
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TestCorrelationEngineBasics:
|
|
31
|
+
def test_rejects_invalid_method(self, census_df):
|
|
32
|
+
with pytest.raises(InvalidMethodError):
|
|
33
|
+
CorrelationEngine(census_df, method="not_a_real_method")
|
|
34
|
+
|
|
35
|
+
def test_rejects_insufficient_numeric_columns(self):
|
|
36
|
+
df = pd.DataFrame({"only_col": [1, 2, 3], "text_col": ["a", "b", "c"]})
|
|
37
|
+
with pytest.raises(InsufficientDataError):
|
|
38
|
+
CorrelationEngine(df)
|
|
39
|
+
|
|
40
|
+
def test_ignores_non_numeric_columns(self, census_df):
|
|
41
|
+
engine = CorrelationEngine(census_df)
|
|
42
|
+
assert "Province" not in engine.columns
|
|
43
|
+
assert "District" not in engine.columns
|
|
44
|
+
assert "Total Population" in engine.columns
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TestMatrix:
|
|
48
|
+
def test_matrix_diagonal_is_one(self, census_df):
|
|
49
|
+
engine = CorrelationEngine(census_df, method="pearson")
|
|
50
|
+
matrix = engine.matrix()
|
|
51
|
+
for col in engine.columns:
|
|
52
|
+
assert matrix.loc[col, col] == pytest.approx(1.0)
|
|
53
|
+
|
|
54
|
+
def test_matrix_is_symmetric(self, census_df):
|
|
55
|
+
engine = CorrelationEngine(census_df, method="spearman")
|
|
56
|
+
matrix = engine.matrix()
|
|
57
|
+
assert matrix.equals(matrix.T)
|
|
58
|
+
|
|
59
|
+
def test_perfect_positive_and_negative_correlation(self, perfectly_correlated_df):
|
|
60
|
+
engine = CorrelationEngine(perfectly_correlated_df, method="pearson")
|
|
61
|
+
matrix = engine.matrix()
|
|
62
|
+
assert matrix.loc["a", "b"] == pytest.approx(1.0)
|
|
63
|
+
assert matrix.loc["a", "c"] == pytest.approx(-1.0)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class TestMatrixPairwiseConsistency:
|
|
67
|
+
"""Regression tests: matrix(), pvalue_matrix(), and pairwise() must
|
|
68
|
+
all apply the same MIN_OBSERVATIONS floor, so they never disagree
|
|
69
|
+
about whether a given pair's coefficient is meaningful."""
|
|
70
|
+
|
|
71
|
+
def test_matrix_is_nan_below_min_observations(self):
|
|
72
|
+
# Only 2 rows — below MIN_OBSERVATIONS (3), so this must NOT
|
|
73
|
+
# silently produce a trivial +-1.0 the way a bare df.corr() would.
|
|
74
|
+
df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
|
|
75
|
+
engine = CorrelationEngine(df, method="pearson")
|
|
76
|
+
matrix = engine.matrix()
|
|
77
|
+
assert pd.isna(matrix.loc["a", "b"])
|
|
78
|
+
assert pd.isna(matrix.loc["b", "a"])
|
|
79
|
+
|
|
80
|
+
def test_matrix_agrees_with_pairwise_below_min_observations(self):
|
|
81
|
+
df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
|
|
82
|
+
engine = CorrelationEngine(df, method="pearson")
|
|
83
|
+
matrix_value = engine.matrix().loc["a", "b"]
|
|
84
|
+
pairwise_value = engine.pairwise("a", "b")["coefficient"]
|
|
85
|
+
assert pd.isna(matrix_value) and pd.isna(pairwise_value)
|
|
86
|
+
|
|
87
|
+
def test_matrix_diagonal_is_one_even_below_min_observations(self):
|
|
88
|
+
# Self-correlation is a definitional identity, not a statistical
|
|
89
|
+
# claim — it stays 1.0 regardless of n.
|
|
90
|
+
df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
|
|
91
|
+
engine = CorrelationEngine(df, method="pearson")
|
|
92
|
+
matrix = engine.matrix()
|
|
93
|
+
assert matrix.loc["a", "a"] == 1.0
|
|
94
|
+
assert matrix.loc["b", "b"] == 1.0
|
|
95
|
+
|
|
96
|
+
def test_matrix_matches_pairwise_above_min_observations(self, census_df):
|
|
97
|
+
engine = CorrelationEngine(census_df, method="pearson")
|
|
98
|
+
matrix = engine.matrix()
|
|
99
|
+
for col_a in engine.columns:
|
|
100
|
+
for col_b in engine.columns:
|
|
101
|
+
if col_a == col_b:
|
|
102
|
+
continue
|
|
103
|
+
pairwise_value = engine.pairwise(col_a, col_b)["coefficient"]
|
|
104
|
+
assert matrix.loc[col_a, col_b] == pytest.approx(pairwise_value, nan_ok=True)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class TestPvalueMatrix:
|
|
108
|
+
def test_pvalue_diagonal_is_zero(self, census_df):
|
|
109
|
+
engine = CorrelationEngine(census_df)
|
|
110
|
+
pvals = engine.pvalue_matrix()
|
|
111
|
+
for col in engine.columns:
|
|
112
|
+
assert pvals.loc[col, col] == 0.0
|
|
113
|
+
|
|
114
|
+
def test_pvalue_matrix_symmetric(self, census_df):
|
|
115
|
+
engine = CorrelationEngine(census_df)
|
|
116
|
+
pvals = engine.pvalue_matrix()
|
|
117
|
+
assert pvals.equals(pvals.T)
|
|
118
|
+
|
|
119
|
+
def test_perfect_correlation_has_near_zero_pvalue(self, perfectly_correlated_df):
|
|
120
|
+
engine = CorrelationEngine(perfectly_correlated_df, method="pearson")
|
|
121
|
+
pvals = engine.pvalue_matrix()
|
|
122
|
+
assert pvals.loc["a", "b"] < 0.01
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TestPairwise:
|
|
126
|
+
def test_pairwise_matches_matrix(self, census_df):
|
|
127
|
+
engine = CorrelationEngine(census_df, method="pearson")
|
|
128
|
+
matrix = engine.matrix()
|
|
129
|
+
result = engine.pairwise("Total Population", "Households")
|
|
130
|
+
assert result["coefficient"] == pytest.approx(matrix.loc["Total Population", "Households"])
|
|
131
|
+
assert result["n"] == len(census_df)
|
|
132
|
+
|
|
133
|
+
def test_pairwise_correlation_function_all_methods(self, perfectly_correlated_df):
|
|
134
|
+
for method in ("pearson", "spearman", "kendall"):
|
|
135
|
+
result = pairwise_correlation(
|
|
136
|
+
perfectly_correlated_df["a"], perfectly_correlated_df["b"], method=method
|
|
137
|
+
)
|
|
138
|
+
assert result["coefficient"] == pytest.approx(1.0)
|
|
139
|
+
assert result["method"] == method
|
|
140
|
+
|
|
141
|
+
def test_pairwise_correlation_rejects_invalid_method(self, perfectly_correlated_df):
|
|
142
|
+
with pytest.raises(InvalidMethodError):
|
|
143
|
+
pairwise_correlation(perfectly_correlated_df["a"], perfectly_correlated_df["b"], method="bogus")
|
|
144
|
+
|
|
145
|
+
def test_handles_missing_values_pairwise(self):
|
|
146
|
+
a = pd.Series([1, 2, None, 4, 5])
|
|
147
|
+
b = pd.Series([2, 4, 6, 8, 10])
|
|
148
|
+
result = pairwise_correlation(a, b, method="pearson")
|
|
149
|
+
assert result["n"] == 4 # the row with a None in `a` is dropped
|
|
150
|
+
assert result["coefficient"] == pytest.approx(1.0)
|
|
151
|
+
|
|
152
|
+
def test_too_few_observations_returns_nan_not_error(self):
|
|
153
|
+
a = pd.Series([1, 2])
|
|
154
|
+
b = pd.Series([3, 4])
|
|
155
|
+
result = pairwise_correlation(a, b, method="pearson")
|
|
156
|
+
assert pd.isna(result["coefficient"])
|
|
157
|
+
assert pd.isna(result["p_value"])
|
|
158
|
+
assert result["n"] == 2
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class TestReport:
|
|
162
|
+
def test_report_sorted_by_strength_descending(self, census_df):
|
|
163
|
+
engine = CorrelationEngine(census_df, method="pearson")
|
|
164
|
+
report = engine.report()
|
|
165
|
+
coefficients = report["coefficient"].abs().tolist()
|
|
166
|
+
assert coefficients == sorted(coefficients, reverse=True)
|
|
167
|
+
|
|
168
|
+
def test_report_one_row_per_unique_pair(self, census_df):
|
|
169
|
+
engine = CorrelationEngine(census_df)
|
|
170
|
+
n = len(engine.columns)
|
|
171
|
+
expected_pairs = n * (n - 1) // 2
|
|
172
|
+
report = engine.report()
|
|
173
|
+
assert len(report) == expected_pairs
|
|
174
|
+
|
|
175
|
+
def test_report_threshold_filter(self, perfectly_correlated_df):
|
|
176
|
+
engine = CorrelationEngine(perfectly_correlated_df)
|
|
177
|
+
report = engine.report(threshold=0.99)
|
|
178
|
+
assert (report["coefficient"].abs() >= 0.99).all()
|
|
179
|
+
|
|
180
|
+
def test_report_significant_only_filter(self, census_df):
|
|
181
|
+
engine = CorrelationEngine(census_df)
|
|
182
|
+
report = engine.report(significant_only=True, alpha=0.05)
|
|
183
|
+
assert (report["p_value"] < 0.05).all()
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for interactive.py, using unittest.mock to stand in for
|
|
3
|
+
questionary's .ask() calls — no real terminal/TTY needed.
|
|
4
|
+
|
|
5
|
+
The prompt_for_export tests specifically guard against a real bug
|
|
6
|
+
found during manual testing: questionary.Choice(value=None) is treated
|
|
7
|
+
by questionary as "no value provided", which makes it default to the
|
|
8
|
+
choice's *title text* rather than a real None. Selecting "No, just
|
|
9
|
+
print to terminal" was therefore silently mistaken for a truthy export
|
|
10
|
+
format, still prompting for a path and printing a false "Exported to"
|
|
11
|
+
message. The fix uses a distinct sentinel ("none") instead of None.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from unittest.mock import patch, MagicMock
|
|
15
|
+
|
|
16
|
+
from tlf_correlation_engine import interactive
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _mock_ask(return_value):
|
|
20
|
+
"""Builds a MagicMock standing in for a questionary prompt object,
|
|
21
|
+
whose .ask() returns the given value."""
|
|
22
|
+
mock_prompt = MagicMock()
|
|
23
|
+
mock_prompt.ask.return_value = return_value
|
|
24
|
+
return mock_prompt
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TestPromptForExport:
|
|
28
|
+
def test_selecting_no_export_skips_path_prompt_entirely(self):
|
|
29
|
+
"""Regression test for the value=None bug: choosing 'No, just
|
|
30
|
+
print to terminal' must return (None, None) and must NOT go on
|
|
31
|
+
to call questionary.text() asking for an export path."""
|
|
32
|
+
with patch("questionary.select", return_value=_mock_ask("none")) as mock_select, \
|
|
33
|
+
patch("questionary.text") as mock_text:
|
|
34
|
+
fmt, path = interactive.prompt_for_export()
|
|
35
|
+
|
|
36
|
+
assert fmt is None
|
|
37
|
+
assert path is None
|
|
38
|
+
mock_text.assert_not_called()
|
|
39
|
+
|
|
40
|
+
def test_ctrl_c_during_select_also_returns_none(self):
|
|
41
|
+
"""questionary returns a real None if the user hits Ctrl+C /
|
|
42
|
+
Esc mid-prompt — that must be treated the same as 'no export'."""
|
|
43
|
+
with patch("questionary.select", return_value=_mock_ask(None)), \
|
|
44
|
+
patch("questionary.text") as mock_text:
|
|
45
|
+
fmt, path = interactive.prompt_for_export()
|
|
46
|
+
|
|
47
|
+
assert fmt is None
|
|
48
|
+
assert path is None
|
|
49
|
+
mock_text.assert_not_called()
|
|
50
|
+
|
|
51
|
+
def test_selecting_csv_prompts_for_and_returns_path(self):
|
|
52
|
+
with patch("questionary.select", return_value=_mock_ask("csv")), \
|
|
53
|
+
patch("questionary.text", return_value=_mock_ask("out.csv")) as mock_text:
|
|
54
|
+
fmt, path = interactive.prompt_for_export()
|
|
55
|
+
|
|
56
|
+
assert fmt == "csv"
|
|
57
|
+
assert path == "out.csv"
|
|
58
|
+
mock_text.assert_called_once()
|
|
59
|
+
|
|
60
|
+
def test_selecting_json_prompts_for_and_returns_path(self):
|
|
61
|
+
with patch("questionary.select", return_value=_mock_ask("json")), \
|
|
62
|
+
patch("questionary.text", return_value=_mock_ask("out.json")):
|
|
63
|
+
fmt, path = interactive.prompt_for_export()
|
|
64
|
+
|
|
65
|
+
assert fmt == "json"
|
|
66
|
+
assert path == "out.json"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TestPromptForMethod:
|
|
70
|
+
def test_returns_selected_method_value(self):
|
|
71
|
+
with patch("questionary.select", return_value=_mock_ask("spearman")):
|
|
72
|
+
assert interactive.prompt_for_method() == "spearman"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class TestPromptForOutput:
|
|
76
|
+
def test_returns_selected_output_value(self):
|
|
77
|
+
with patch("questionary.select", return_value=_mock_ask("matrix")):
|
|
78
|
+
assert interactive.prompt_for_output() == "matrix"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class TestPromptForReportFilters:
|
|
82
|
+
def test_no_threshold_when_declined(self):
|
|
83
|
+
with patch("questionary.confirm", side_effect=[_mock_ask(False), _mock_ask(False)]):
|
|
84
|
+
threshold, significant_only = interactive.prompt_for_report_filters()
|
|
85
|
+
|
|
86
|
+
assert threshold is None
|
|
87
|
+
assert significant_only is False
|
|
88
|
+
|
|
89
|
+
def test_threshold_captured_when_accepted(self):
|
|
90
|
+
with patch("questionary.confirm", side_effect=[_mock_ask(True), _mock_ask(True)]), \
|
|
91
|
+
patch("questionary.text", return_value=_mock_ask("0.7")):
|
|
92
|
+
threshold, significant_only = interactive.prompt_for_report_filters()
|
|
93
|
+
|
|
94
|
+
assert threshold == 0.7
|
|
95
|
+
assert significant_only is True
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class TestPromptForSheet:
|
|
99
|
+
def test_single_sheet_returned_without_prompting(self):
|
|
100
|
+
with patch("questionary.select") as mock_select:
|
|
101
|
+
result = interactive.prompt_for_sheet(["OnlySheet"])
|
|
102
|
+
|
|
103
|
+
assert result == "OnlySheet"
|
|
104
|
+
mock_select.assert_not_called()
|
|
105
|
+
|
|
106
|
+
def test_no_sheets_returns_none_without_prompting(self):
|
|
107
|
+
with patch("questionary.select") as mock_select:
|
|
108
|
+
result = interactive.prompt_for_sheet([])
|
|
109
|
+
|
|
110
|
+
assert result is None
|
|
111
|
+
mock_select.assert_not_called()
|
|
112
|
+
|
|
113
|
+
def test_multiple_sheets_prompts_and_returns_choice(self):
|
|
114
|
+
with patch("questionary.select", return_value=_mock_ask("SheetB")) as mock_select:
|
|
115
|
+
result = interactive.prompt_for_sheet(["SheetA", "SheetB"])
|
|
116
|
+
|
|
117
|
+
assert result == "SheetB"
|
|
118
|
+
mock_select.assert_called_once()
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tlf-correlation-engine
|
|
3
|
+
-----------------------
|
|
4
|
+
Country-agnostic correlation analysis (Pearson, Spearman, Kendall) over
|
|
5
|
+
any pandas DataFrame's numeric columns — coefficient matrices, p-values,
|
|
6
|
+
and a sortable/filterable long-form report of relationship strength.
|
|
7
|
+
Part of the TLF-Data-Analysis repo, part of the TLF ("The Living Facts")
|
|
8
|
+
initiative.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from .engine import CorrelationEngine, SUPPORTED_METHODS
|
|
12
|
+
from .methods import pairwise_correlation
|
|
13
|
+
from .loader import TabularLoader, UnsupportedFileError
|
|
14
|
+
from .errors import CorrelationEngineError, InvalidMethodError, InsufficientDataError
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"__version__",
|
|
18
|
+
"CorrelationEngine",
|
|
19
|
+
"SUPPORTED_METHODS",
|
|
20
|
+
"pairwise_correlation",
|
|
21
|
+
"TabularLoader",
|
|
22
|
+
"UnsupportedFileError",
|
|
23
|
+
"CorrelationEngineError",
|
|
24
|
+
"InvalidMethodError",
|
|
25
|
+
"InsufficientDataError",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|