plotaviz 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plotaviz/__init__.py +19 -0
- plotaviz/core/__init__.py +53 -0
- plotaviz/core/analysis.py +230 -0
- plotaviz/core/codegen.py +482 -0
- plotaviz/core/errors.py +65 -0
- plotaviz/core/exporter.py +145 -0
- plotaviz/core/llm/__init__.py +211 -0
- plotaviz/core/llm/anthropic.py +61 -0
- plotaviz/core/llm/base.py +373 -0
- plotaviz/core/llm/gemini.py +51 -0
- plotaviz/core/llm/ollama.py +113 -0
- plotaviz/core/llm/openai.py +58 -0
- plotaviz/core/loader.py +366 -0
- plotaviz/core/plotter.py +790 -0
- plotaviz/core/preprocess.py +836 -0
- plotaviz/core/profiler.py +509 -0
- plotaviz/core/rules.yaml +190 -0
- plotaviz/core/selector.py +495 -0
- plotaviz/core/session.py +174 -0
- plotaviz/core/spec.py +236 -0
- plotaviz/main.py +329 -0
- plotaviz/ui/__init__.py +23 -0
- plotaviz/ui/export_dialog.py +228 -0
- plotaviz/ui/filter_panel.py +273 -0
- plotaviz/ui/interactive_view.py +210 -0
- plotaviz/ui/main_window.py +859 -0
- plotaviz/ui/nl_query_bar.py +219 -0
- plotaviz/ui/settings_dialog.py +313 -0
- plotaviz/ui/table_model.py +184 -0
- plotaviz/ui/type_override.py +159 -0
- plotaviz/ui/workers.py +222 -0
- plotaviz-0.1.0.dist-info/METADATA +410 -0
- plotaviz-0.1.0.dist-info/RECORD +37 -0
- plotaviz-0.1.0.dist-info/WHEEL +5 -0
- plotaviz-0.1.0.dist-info/entry_points.txt +2 -0
- plotaviz-0.1.0.dist-info/licenses/LICENSE +21 -0
- plotaviz-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
"""Column typing and dataset statistics.
|
|
2
|
+
|
|
3
|
+
The profiler answers the question the chart selector needs answered: *what shape is this data?*
|
|
4
|
+
It assigns every column a **role** — numeric, categorical, datetime, boolean, or high-cardinality
|
|
5
|
+
text — and attaches the statistics the scoring layer uses to rank charts (cardinality, missing
|
|
6
|
+
fraction, skew, correlation strength).
|
|
7
|
+
|
|
8
|
+
Roles are deliberately coarser than dtypes. An integer column of 30,000 unique customer IDs is
|
|
9
|
+
numerically typed but is not a *measure*, and charting it as one produces nonsense; the role
|
|
10
|
+
system exists to catch exactly that case.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from .errors import ProfileError
|
|
23
|
+
|
|
24
|
+
# Roles ---------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
NUMERIC = "numeric"
|
|
27
|
+
CATEGORICAL = "categorical"
|
|
28
|
+
DATETIME = "datetime"
|
|
29
|
+
BOOLEAN = "boolean"
|
|
30
|
+
TEXT = "text" # high-cardinality free text — never charted directly
|
|
31
|
+
|
|
32
|
+
ROLES: tuple[str, ...] = (NUMERIC, CATEGORICAL, DATETIME, BOOLEAN, TEXT)
|
|
33
|
+
|
|
34
|
+
#: A column with at most this many distinct values is categorical even if it is numeric.
|
|
35
|
+
LOW_CARDINALITY_MAX = 20
|
|
36
|
+
|
|
37
|
+
#: Above this ratio of distinct values to rows, a string column is free text, not a category.
|
|
38
|
+
TEXT_CARDINALITY_RATIO = 0.5
|
|
39
|
+
|
|
40
|
+
#: A numeric column whose values are this unique is an identifier, not a measure.
|
|
41
|
+
ID_CARDINALITY_RATIO = 0.95
|
|
42
|
+
|
|
43
|
+
#: Rows sampled for profiling when the frame is bigger than this.
|
|
44
|
+
PROFILE_SAMPLE_ROWS = 100_000
|
|
45
|
+
|
|
46
|
+
#: Suffixes marking a column PlotaViz added itself. These are shown in the preview but are never
|
|
47
|
+
#: offered as chart axes — recommending a chart of your own outlier flags is noise.
|
|
48
|
+
DERIVED_SUFFIXES = ("__outlier",)
|
|
49
|
+
|
|
50
|
+
#: Column-name patterns that mark an identifier regardless of dtype.
|
|
51
|
+
_ID_NAME_PATTERN = re.compile(r"(^|_)(id|uuid|guid|key|code|index|no|num)$", re.IGNORECASE)
|
|
52
|
+
|
|
53
|
+
#: Column-name patterns that hint at a date, used to justify a parse attempt.
|
|
54
|
+
_DATE_NAME_PATTERN = re.compile(
|
|
55
|
+
r"(date|time|timestamp|datetime|day|month|year|created|updated|_at$)", re.IGNORECASE
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
_BOOLEAN_TOKENS = (
|
|
59
|
+
{"true", "false"},
|
|
60
|
+
{"yes", "no"},
|
|
61
|
+
{"y", "n"},
|
|
62
|
+
{"t", "f"},
|
|
63
|
+
{"0", "1"},
|
|
64
|
+
{"on", "off"},
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class ColumnProfile:
|
|
70
|
+
"""Everything known about one column.
|
|
71
|
+
|
|
72
|
+
Attributes:
|
|
73
|
+
name: Column name as it appears in the dataframe.
|
|
74
|
+
dtype: The pandas dtype, as a string.
|
|
75
|
+
role: One of :data:`ROLES`.
|
|
76
|
+
n_missing: Count of nulls.
|
|
77
|
+
pct_missing: Nulls as a percentage of rows, 0–100.
|
|
78
|
+
n_unique: Distinct non-null values.
|
|
79
|
+
cardinality_ratio: ``n_unique / n_rows``, the identifier detector's main signal.
|
|
80
|
+
is_identifier: True when the column looks like a key rather than a measure.
|
|
81
|
+
is_derived: True for columns PlotaViz itself added (outlier flags). They are shown in the
|
|
82
|
+
preview table but never offered as chart axes.
|
|
83
|
+
stats: Role-specific numbers — ``min``/``max``/``mean``/``median``/``std``/``skew`` for
|
|
84
|
+
numeric columns, ``min``/``max``/``span_days`` for datetimes.
|
|
85
|
+
top_values: Most frequent values with counts, for categorical and boolean columns.
|
|
86
|
+
sample_values: A few example values, used in the LLM prompt and the UI.
|
|
87
|
+
note: Human-readable remark, e.g. why a numeric column was treated as an identifier.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
name: str
|
|
91
|
+
dtype: str
|
|
92
|
+
role: str
|
|
93
|
+
n_missing: int = 0
|
|
94
|
+
pct_missing: float = 0.0
|
|
95
|
+
n_unique: int = 0
|
|
96
|
+
cardinality_ratio: float = 0.0
|
|
97
|
+
is_identifier: bool = False
|
|
98
|
+
is_derived: bool = False
|
|
99
|
+
stats: dict[str, float] = field(default_factory=dict)
|
|
100
|
+
top_values: list[tuple[Any, int]] = field(default_factory=list)
|
|
101
|
+
sample_values: list[Any] = field(default_factory=list)
|
|
102
|
+
note: str = ""
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def is_low_cardinality(self) -> bool:
|
|
106
|
+
"""Whether this column is small enough to make a readable categorical axis."""
|
|
107
|
+
return self.n_unique <= LOW_CARDINALITY_MAX
|
|
108
|
+
|
|
109
|
+
def to_dict(self) -> dict[str, Any]:
|
|
110
|
+
"""JSON-safe view, used for session files and LLM prompts."""
|
|
111
|
+
return {
|
|
112
|
+
"name": self.name,
|
|
113
|
+
"dtype": self.dtype,
|
|
114
|
+
"role": self.role,
|
|
115
|
+
"pct_missing": round(self.pct_missing, 2),
|
|
116
|
+
"n_unique": self.n_unique,
|
|
117
|
+
"is_identifier": self.is_identifier,
|
|
118
|
+
"stats": {k: _json_safe(v) for k, v in self.stats.items()},
|
|
119
|
+
"top_values": [[_json_safe(v), c] for v, c in self.top_values[:5]],
|
|
120
|
+
"sample_values": [_json_safe(v) for v in self.sample_values[:5]],
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class DatasetProfile:
|
|
126
|
+
"""Aggregate view of a dataset: every column profile plus frame-level statistics.
|
|
127
|
+
|
|
128
|
+
Attributes:
|
|
129
|
+
n_rows: Rows profiled.
|
|
130
|
+
n_cols: Column count.
|
|
131
|
+
columns: Profiles keyed by column name, in dataframe order.
|
|
132
|
+
n_duplicate_rows: Fully duplicated rows found.
|
|
133
|
+
correlations: Pearson correlation matrix over numeric non-identifier columns, or ``None``
|
|
134
|
+
when there are fewer than two of them.
|
|
135
|
+
sampled: Whether profiling ran on a sample.
|
|
136
|
+
total_rows: Rows in the source, which exceeds :attr:`n_rows` when sampled.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
n_rows: int
|
|
140
|
+
n_cols: int
|
|
141
|
+
columns: dict[str, ColumnProfile]
|
|
142
|
+
n_duplicate_rows: int = 0
|
|
143
|
+
correlations: pd.DataFrame | None = None
|
|
144
|
+
sampled: bool = False
|
|
145
|
+
total_rows: int = 0
|
|
146
|
+
|
|
147
|
+
# -------------------------------------------------------------- role queries
|
|
148
|
+
|
|
149
|
+
def by_role(
|
|
150
|
+
self, role: str, *, include_identifiers: bool = False, include_derived: bool = False
|
|
151
|
+
) -> list[str]:
|
|
152
|
+
"""Column names with the given role, in dataframe order.
|
|
153
|
+
|
|
154
|
+
Identifiers and PlotaViz-generated columns are excluded by default — detecting them is
|
|
155
|
+
pointless if they still end up on an axis.
|
|
156
|
+
"""
|
|
157
|
+
return [
|
|
158
|
+
name
|
|
159
|
+
for name, prof in self.columns.items()
|
|
160
|
+
if prof.role == role
|
|
161
|
+
and (include_identifiers or not prof.is_identifier)
|
|
162
|
+
and (include_derived or not prof.is_derived)
|
|
163
|
+
]
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def numeric(self) -> list[str]:
|
|
167
|
+
"""Numeric measure columns (identifiers excluded)."""
|
|
168
|
+
return self.by_role(NUMERIC)
|
|
169
|
+
|
|
170
|
+
@property
|
|
171
|
+
def categorical(self) -> list[str]:
|
|
172
|
+
"""Categorical columns (identifiers excluded)."""
|
|
173
|
+
return self.by_role(CATEGORICAL)
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def datetime(self) -> list[str]:
|
|
177
|
+
"""Datetime columns."""
|
|
178
|
+
return self.by_role(DATETIME)
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def boolean(self) -> list[str]:
|
|
182
|
+
"""Boolean columns."""
|
|
183
|
+
return self.by_role(BOOLEAN)
|
|
184
|
+
|
|
185
|
+
@property
|
|
186
|
+
def text(self) -> list[str]:
|
|
187
|
+
"""High-cardinality text columns."""
|
|
188
|
+
return self.by_role(TEXT)
|
|
189
|
+
|
|
190
|
+
def role_counts(self) -> dict[str, int]:
|
|
191
|
+
"""How many chartable columns exist per role — the selector's rule-matching input."""
|
|
192
|
+
return {role: len(self.by_role(role)) for role in ROLES}
|
|
193
|
+
|
|
194
|
+
def strongest_correlation(self) -> tuple[str, str, float] | None:
|
|
195
|
+
"""The most strongly correlated numeric pair, as ``(col_a, col_b, r)``.
|
|
196
|
+
|
|
197
|
+
Returns ``None`` when there is no correlation matrix. The sign is preserved; the pair is
|
|
198
|
+
chosen by absolute value.
|
|
199
|
+
"""
|
|
200
|
+
if self.correlations is None or self.correlations.shape[0] < 2:
|
|
201
|
+
return None
|
|
202
|
+
corr = self.correlations
|
|
203
|
+
best: tuple[str, str, float] | None = None
|
|
204
|
+
cols = list(corr.columns)
|
|
205
|
+
for i, a in enumerate(cols):
|
|
206
|
+
for b in cols[i + 1 :]:
|
|
207
|
+
value = as_float(corr.loc[a, b])
|
|
208
|
+
if value is None:
|
|
209
|
+
continue
|
|
210
|
+
if best is None or abs(value) > abs(best[2]):
|
|
211
|
+
best = (a, b, value)
|
|
212
|
+
return best
|
|
213
|
+
|
|
214
|
+
def schema_summary(self) -> dict[str, Any]:
|
|
215
|
+
"""Compact schema + statistics payload.
|
|
216
|
+
|
|
217
|
+
This is exactly what gets sent to a language model — schema, summary statistics, and a
|
|
218
|
+
handful of sample values. The dataset itself never leaves the machine.
|
|
219
|
+
"""
|
|
220
|
+
return {
|
|
221
|
+
"n_rows": self.total_rows or self.n_rows,
|
|
222
|
+
"n_cols": self.n_cols,
|
|
223
|
+
"sampled": self.sampled,
|
|
224
|
+
"columns": [prof.to_dict() for prof in self.columns.values()],
|
|
225
|
+
"strongest_correlation": self.strongest_correlation(),
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# ---------------------------------------------------------------------------- inference
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def looks_like_datetime(
|
|
233
|
+
series: pd.Series, *, name_hint: bool = True, threshold: float = 0.8
|
|
234
|
+
) -> bool:
|
|
235
|
+
"""Whether a string/object series parses cleanly as dates.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
series: The column to test.
|
|
239
|
+
name_hint: Whether the column name is allowed to lower the bar. Date-named columns get a
|
|
240
|
+
parse attempt even when the first values are ambiguous.
|
|
241
|
+
threshold: Fraction of non-null values that must parse for the answer to be yes.
|
|
242
|
+
|
|
243
|
+
Pandas will happily coerce plain integers into 1970-era nanosecond timestamps, so numeric
|
|
244
|
+
columns are rejected outright regardless of their name.
|
|
245
|
+
"""
|
|
246
|
+
if pd.api.types.is_datetime64_any_dtype(series):
|
|
247
|
+
return True
|
|
248
|
+
if pd.api.types.is_numeric_dtype(series) or pd.api.types.is_bool_dtype(series):
|
|
249
|
+
return False
|
|
250
|
+
|
|
251
|
+
non_null = series.dropna()
|
|
252
|
+
if non_null.empty:
|
|
253
|
+
return False
|
|
254
|
+
|
|
255
|
+
sample = non_null.head(1000).astype(str)
|
|
256
|
+
if name_hint and _DATE_NAME_PATTERN.search(str(series.name or "")):
|
|
257
|
+
threshold = min(threshold, 0.6)
|
|
258
|
+
|
|
259
|
+
parsed = pd.to_datetime(sample, errors="coerce", format="mixed")
|
|
260
|
+
return float(parsed.notna().mean()) >= threshold
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def looks_like_boolean(series: pd.Series) -> bool:
|
|
264
|
+
"""Whether a two-valued column represents a boolean rather than a small category."""
|
|
265
|
+
if pd.api.types.is_bool_dtype(series):
|
|
266
|
+
return True
|
|
267
|
+
non_null = series.dropna()
|
|
268
|
+
if non_null.empty:
|
|
269
|
+
return False
|
|
270
|
+
values = {str(v).strip().lower() for v in non_null.unique()[:10]}
|
|
271
|
+
return len(values) <= 2 and any(values <= tokens for tokens in _BOOLEAN_TOKENS)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _is_key_like(values: pd.Series) -> bool:
|
|
275
|
+
"""Whether a near-unique integer column occupies a dense, contiguous range.
|
|
276
|
+
|
|
277
|
+
Row identifiers and auto-increment keys sit in a narrow band — 1..n, or 1000..1500. Genuine
|
|
278
|
+
measures that happen to be all-unique, like a city's population, are spread across orders of
|
|
279
|
+
magnitude. Without this check, every unique-valued measure gets misfiled as an identifier and
|
|
280
|
+
silently excluded from every chart.
|
|
281
|
+
"""
|
|
282
|
+
numbers = pd.to_numeric(values, errors="coerce").dropna()
|
|
283
|
+
if numbers.empty:
|
|
284
|
+
return False
|
|
285
|
+
span = float(numbers.max() - numbers.min()) + 1.0
|
|
286
|
+
return span <= 2.0 * len(numbers)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def infer_role(series: pd.Series, *, n_rows: int | None = None) -> tuple[str, bool, str]:
|
|
290
|
+
"""Classify one column.
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
``(role, is_identifier, note)`` where ``note`` explains any non-obvious decision so the
|
|
294
|
+
UI can show the user *why* their integer column is not being treated as a measure.
|
|
295
|
+
"""
|
|
296
|
+
n_rows = n_rows or len(series)
|
|
297
|
+
non_null = series.dropna()
|
|
298
|
+
n_unique = int(non_null.nunique())
|
|
299
|
+
ratio = n_unique / n_rows if n_rows else 0.0
|
|
300
|
+
name = str(series.name or "")
|
|
301
|
+
name_says_id = bool(_ID_NAME_PATTERN.search(name))
|
|
302
|
+
|
|
303
|
+
if non_null.empty:
|
|
304
|
+
return CATEGORICAL, False, "Column is entirely empty."
|
|
305
|
+
|
|
306
|
+
if looks_like_boolean(series):
|
|
307
|
+
return BOOLEAN, False, ""
|
|
308
|
+
|
|
309
|
+
if pd.api.types.is_datetime64_any_dtype(series):
|
|
310
|
+
return DATETIME, False, ""
|
|
311
|
+
|
|
312
|
+
if pd.api.types.is_numeric_dtype(series):
|
|
313
|
+
is_integral = (
|
|
314
|
+
pd.api.types.is_integer_dtype(series) or (non_null.astype(float) % 1 == 0).all()
|
|
315
|
+
)
|
|
316
|
+
if name_says_id and is_integral:
|
|
317
|
+
return NUMERIC, True, f"Treated {name!r} as an identifier because of its name."
|
|
318
|
+
if is_integral and ratio >= ID_CARDINALITY_RATIO and n_rows > 20 and _is_key_like(non_null):
|
|
319
|
+
return (
|
|
320
|
+
NUMERIC,
|
|
321
|
+
True,
|
|
322
|
+
f"{name!r} is a nearly all-unique run of consecutive whole numbers, so it looks "
|
|
323
|
+
"like a row identifier.",
|
|
324
|
+
)
|
|
325
|
+
if is_integral and n_unique <= LOW_CARDINALITY_MAX and n_unique <= 12:
|
|
326
|
+
return (
|
|
327
|
+
CATEGORICAL,
|
|
328
|
+
False,
|
|
329
|
+
f"{name!r} has only {n_unique} distinct whole numbers, so it reads as a category.",
|
|
330
|
+
)
|
|
331
|
+
return NUMERIC, False, ""
|
|
332
|
+
|
|
333
|
+
# Object / string columns.
|
|
334
|
+
if looks_like_datetime(series):
|
|
335
|
+
return DATETIME, False, f"Parsed {name!r} from text into dates."
|
|
336
|
+
|
|
337
|
+
if (name_says_id or ratio >= ID_CARDINALITY_RATIO) and ratio >= TEXT_CARDINALITY_RATIO:
|
|
338
|
+
return (
|
|
339
|
+
TEXT,
|
|
340
|
+
True,
|
|
341
|
+
f"{name!r} is nearly all-unique text, so it looks like an identifier.",
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
if ratio >= TEXT_CARDINALITY_RATIO and n_unique > LOW_CARDINALITY_MAX:
|
|
345
|
+
return TEXT, False, f"{name!r} is high-cardinality free text ({n_unique:,} values)."
|
|
346
|
+
|
|
347
|
+
return CATEGORICAL, False, ""
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# ---------------------------------------------------------------------------- profiling
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def profile(
|
|
354
|
+
df: pd.DataFrame,
|
|
355
|
+
*,
|
|
356
|
+
overrides: dict[str, str] | None = None,
|
|
357
|
+
sample_rows: int = PROFILE_SAMPLE_ROWS,
|
|
358
|
+
total_rows: int | None = None,
|
|
359
|
+
) -> DatasetProfile:
|
|
360
|
+
"""Profile a dataframe.
|
|
361
|
+
|
|
362
|
+
Args:
|
|
363
|
+
df: The data to profile.
|
|
364
|
+
overrides: User-supplied ``{column: role}`` corrections from the type override panel.
|
|
365
|
+
These win over inference unconditionally — auto-inference gets IDs and dates wrong
|
|
366
|
+
constantly, and the user is the authority.
|
|
367
|
+
sample_rows: Profile on a random sample above this row count. Statistics stay
|
|
368
|
+
representative and the UI stays responsive.
|
|
369
|
+
total_rows: True source row count when ``df`` is already a sample.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
A :class:`DatasetProfile`.
|
|
373
|
+
|
|
374
|
+
Raises:
|
|
375
|
+
ProfileError: If the frame has no rows or no columns.
|
|
376
|
+
"""
|
|
377
|
+
if df is None or df.empty:
|
|
378
|
+
raise ProfileError("There is no data to profile.")
|
|
379
|
+
if len(df.columns) == 0:
|
|
380
|
+
raise ProfileError("The dataset has no columns.")
|
|
381
|
+
|
|
382
|
+
overrides = overrides or {}
|
|
383
|
+
sampled = len(df) > sample_rows
|
|
384
|
+
frame = df.sample(sample_rows, random_state=0) if sampled else df
|
|
385
|
+
n_rows = len(frame)
|
|
386
|
+
|
|
387
|
+
columns: dict[str, ColumnProfile] = {}
|
|
388
|
+
for name in df.columns:
|
|
389
|
+
series = frame[name]
|
|
390
|
+
role, is_id, note = infer_role(series, n_rows=n_rows)
|
|
391
|
+
|
|
392
|
+
override = overrides.get(str(name))
|
|
393
|
+
if override in ROLES:
|
|
394
|
+
if override != role:
|
|
395
|
+
note = f"Type set to {override} by you (inferred: {role})."
|
|
396
|
+
role, is_id = override, False
|
|
397
|
+
if override == DATETIME and not pd.api.types.is_datetime64_any_dtype(series):
|
|
398
|
+
series = pd.to_datetime(series, errors="coerce", format="mixed")
|
|
399
|
+
elif override == NUMERIC and not pd.api.types.is_numeric_dtype(series):
|
|
400
|
+
series = pd.to_numeric(series, errors="coerce")
|
|
401
|
+
|
|
402
|
+
n_missing = int(series.isna().sum())
|
|
403
|
+
n_unique = int(series.dropna().nunique())
|
|
404
|
+
prof = ColumnProfile(
|
|
405
|
+
name=str(name),
|
|
406
|
+
dtype=str(df[name].dtype),
|
|
407
|
+
role=role,
|
|
408
|
+
n_missing=n_missing,
|
|
409
|
+
pct_missing=100.0 * n_missing / n_rows if n_rows else 0.0,
|
|
410
|
+
n_unique=n_unique,
|
|
411
|
+
cardinality_ratio=n_unique / n_rows if n_rows else 0.0,
|
|
412
|
+
is_identifier=is_id,
|
|
413
|
+
is_derived=str(name).endswith(DERIVED_SUFFIXES),
|
|
414
|
+
note=note,
|
|
415
|
+
sample_values=[_json_safe(v) for v in series.dropna().head(5).tolist()],
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
if role == NUMERIC:
|
|
419
|
+
prof.stats = _numeric_stats(series)
|
|
420
|
+
elif role == DATETIME:
|
|
421
|
+
prof.stats = _datetime_stats(series)
|
|
422
|
+
if role in (CATEGORICAL, BOOLEAN) or (role == TEXT and n_unique <= 50):
|
|
423
|
+
counts = series.dropna().value_counts().head(10)
|
|
424
|
+
prof.top_values = [(_json_safe(idx), int(cnt)) for idx, cnt in counts.items()]
|
|
425
|
+
|
|
426
|
+
columns[str(name)] = prof
|
|
427
|
+
|
|
428
|
+
correlations = _correlations(frame, columns)
|
|
429
|
+
|
|
430
|
+
return DatasetProfile(
|
|
431
|
+
n_rows=n_rows,
|
|
432
|
+
n_cols=len(df.columns),
|
|
433
|
+
columns=columns,
|
|
434
|
+
n_duplicate_rows=int(frame.duplicated().sum()),
|
|
435
|
+
correlations=correlations,
|
|
436
|
+
sampled=sampled,
|
|
437
|
+
total_rows=total_rows or len(df),
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _numeric_stats(series: pd.Series) -> dict[str, float]:
|
|
442
|
+
"""Min/max/mean/median/std/skew for a numeric column, all as plain floats."""
|
|
443
|
+
values = pd.to_numeric(series, errors="coerce").dropna()
|
|
444
|
+
if values.empty:
|
|
445
|
+
return {}
|
|
446
|
+
skew = as_float(values.skew()) or 0.0
|
|
447
|
+
return {
|
|
448
|
+
"min": float(values.min()),
|
|
449
|
+
"max": float(values.max()),
|
|
450
|
+
"mean": float(values.mean()),
|
|
451
|
+
"median": float(values.median()),
|
|
452
|
+
"std": float(values.std()) if len(values) > 1 else 0.0,
|
|
453
|
+
"skew": 0.0 if np.isnan(skew) else skew,
|
|
454
|
+
"zeros": float((values == 0).sum()),
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _datetime_stats(series: pd.Series) -> dict[str, float]:
|
|
459
|
+
"""Span statistics for a datetime column, expressed in days."""
|
|
460
|
+
values = pd.to_datetime(series, errors="coerce").dropna()
|
|
461
|
+
if values.empty:
|
|
462
|
+
return {}
|
|
463
|
+
span = values.max() - values.min()
|
|
464
|
+
return {
|
|
465
|
+
"span_days": float(span.total_seconds() / 86400.0),
|
|
466
|
+
"n_periods": float(values.nunique()),
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _correlations(df: pd.DataFrame, columns: dict[str, ColumnProfile]) -> pd.DataFrame | None:
|
|
471
|
+
"""Pearson correlations across numeric measure columns, or ``None`` if fewer than two."""
|
|
472
|
+
numeric = [
|
|
473
|
+
name for name, prof in columns.items() if prof.role == NUMERIC and not prof.is_identifier
|
|
474
|
+
]
|
|
475
|
+
if len(numeric) < 2:
|
|
476
|
+
return None
|
|
477
|
+
try:
|
|
478
|
+
return df[numeric].apply(pd.to_numeric, errors="coerce").corr(numeric_only=True)
|
|
479
|
+
except Exception:
|
|
480
|
+
return None
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def as_float(value: Any) -> float | None:
|
|
484
|
+
"""Coerce a pandas scalar to a plain float, or ``None`` if it is not numeric.
|
|
485
|
+
|
|
486
|
+
Pandas scalars are typed as a broad union (timestamps, timedeltas, bytes …), so calling
|
|
487
|
+
``float()`` on one directly is both a type error and a real ``TypeError`` risk on odd data.
|
|
488
|
+
"""
|
|
489
|
+
if value is None or pd.isna(value):
|
|
490
|
+
return None
|
|
491
|
+
try:
|
|
492
|
+
return float(value) # type: ignore[arg-type]
|
|
493
|
+
except (TypeError, ValueError):
|
|
494
|
+
return None
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _json_safe(value: Any) -> Any:
|
|
498
|
+
"""Coerce numpy/pandas scalars into plain Python types for JSON serialization."""
|
|
499
|
+
if value is None or isinstance(value, (str, bool, int, float)):
|
|
500
|
+
return value
|
|
501
|
+
if isinstance(value, (np.integer,)):
|
|
502
|
+
return int(value)
|
|
503
|
+
if isinstance(value, (np.floating,)):
|
|
504
|
+
return float(value)
|
|
505
|
+
if isinstance(value, (np.bool_,)):
|
|
506
|
+
return bool(value)
|
|
507
|
+
if isinstance(value, (pd.Timestamp,)):
|
|
508
|
+
return value.isoformat()
|
|
509
|
+
return str(value)
|
plotaviz/core/rules.yaml
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# PlotaViz chart selection rules.
|
|
2
|
+
#
|
|
3
|
+
# This file is the editable half of the selection engine. `selector.py` reads it, matches each
|
|
4
|
+
# rule against the dataset's role counts, expands the column placeholders into concrete chart
|
|
5
|
+
# specs, and then scores them. Adding a chart usually means adding a rule here, not writing code.
|
|
6
|
+
#
|
|
7
|
+
# Placeholder syntax in `map`:
|
|
8
|
+
# numeric.0 the first numeric column, numeric.1 the second, and so on
|
|
9
|
+
# categorical.0 likewise for categorical / datetime / boolean / text
|
|
10
|
+
# ?categorical.0 optional — the candidate is still produced if no such column exists
|
|
11
|
+
#
|
|
12
|
+
# Columns are ordered by usefulness (identifiers excluded, low-missing and low-cardinality
|
|
13
|
+
# first), so `.0` means "the most promising column of that role".
|
|
14
|
+
|
|
15
|
+
version: 1
|
|
16
|
+
|
|
17
|
+
scoring:
|
|
18
|
+
# Readability caps. Charts exceeding these are penalized, not forbidden — the top-N option in
|
|
19
|
+
# the plotter can often rescue them.
|
|
20
|
+
max_bar_categories: 30
|
|
21
|
+
max_pie_categories: 6
|
|
22
|
+
max_grouped_series: 8
|
|
23
|
+
max_treemap_categories: 200
|
|
24
|
+
ideal_hist_rows: 30
|
|
25
|
+
|
|
26
|
+
weights:
|
|
27
|
+
# Multiplied by the normalized signal and added to the base score.
|
|
28
|
+
cardinality_penalty: 0.35 # categories beyond the cap
|
|
29
|
+
missing_penalty: 0.25 # fraction of missing values in the mapped columns
|
|
30
|
+
correlation_bonus: 0.25 # |r| for scatter-type charts
|
|
31
|
+
skew_bonus: 0.15 # skewed numerics favour box/violin and log scales
|
|
32
|
+
timespan_bonus: 0.18 # a wide, well-populated date range favours time series
|
|
33
|
+
identifier_penalty: 0.60 # mapping an identifier column is almost always wrong
|
|
34
|
+
constant_penalty: 0.50 # a single-valued column carries no information
|
|
35
|
+
|
|
36
|
+
rules:
|
|
37
|
+
# ---- one numeric column ------------------------------------------------
|
|
38
|
+
- name: single_numeric
|
|
39
|
+
requires: { numeric: 1 }
|
|
40
|
+
candidates:
|
|
41
|
+
- chart: histogram
|
|
42
|
+
base: 0.88
|
|
43
|
+
map: { x: numeric.0 }
|
|
44
|
+
why: "One numeric column, so a histogram shows how its values are distributed."
|
|
45
|
+
- chart: kde
|
|
46
|
+
base: 0.62
|
|
47
|
+
map: { x: numeric.0 }
|
|
48
|
+
why: "A density curve smooths the distribution and reads well for continuous values."
|
|
49
|
+
- chart: box
|
|
50
|
+
base: 0.55
|
|
51
|
+
map: { x: "?categorical.0", y: numeric.0 }
|
|
52
|
+
why: "A box plot summarizes spread and exposes outliers at a glance."
|
|
53
|
+
|
|
54
|
+
# ---- one categorical column --------------------------------------------
|
|
55
|
+
- name: single_categorical
|
|
56
|
+
requires: { categorical: 1 }
|
|
57
|
+
candidates:
|
|
58
|
+
- chart: bar
|
|
59
|
+
base: 0.85
|
|
60
|
+
map: { x: categorical.0 }
|
|
61
|
+
agg: count
|
|
62
|
+
why: "One categorical column, so a bar chart of counts compares the groups directly."
|
|
63
|
+
- chart: treemap
|
|
64
|
+
base: 0.45
|
|
65
|
+
map: { x: categorical.0 }
|
|
66
|
+
agg: count
|
|
67
|
+
why: "With many categories a treemap stays readable where a bar chart would not."
|
|
68
|
+
- chart: pie
|
|
69
|
+
base: 0.30
|
|
70
|
+
map: { x: categorical.0 }
|
|
71
|
+
agg: count
|
|
72
|
+
why: "A pie chart works for a handful of categories that make up a whole."
|
|
73
|
+
|
|
74
|
+
# ---- two numeric columns ------------------------------------------------
|
|
75
|
+
- name: numeric_pair
|
|
76
|
+
requires: { numeric: 2 }
|
|
77
|
+
candidates:
|
|
78
|
+
- chart: scatter
|
|
79
|
+
base: 0.78
|
|
80
|
+
map: { x: numeric.0, y: numeric.1, color: "?categorical.0" }
|
|
81
|
+
options: { trendline: true }
|
|
82
|
+
why: "Two numeric columns, so a scatter plot shows how they relate."
|
|
83
|
+
|
|
84
|
+
# ---- numeric by category ------------------------------------------------
|
|
85
|
+
- name: numeric_by_category
|
|
86
|
+
requires: { numeric: 1, categorical: 1 }
|
|
87
|
+
candidates:
|
|
88
|
+
- chart: bar
|
|
89
|
+
base: 0.86
|
|
90
|
+
map: { x: categorical.0, y: numeric.0 }
|
|
91
|
+
agg: mean
|
|
92
|
+
why: "Comparing an average measure across categories is what a bar chart is for."
|
|
93
|
+
- chart: box
|
|
94
|
+
base: 0.80
|
|
95
|
+
map: { x: categorical.0, y: numeric.0 }
|
|
96
|
+
why: "A box plot compares the full distribution per category, not just the average."
|
|
97
|
+
- chart: violin
|
|
98
|
+
base: 0.66
|
|
99
|
+
map: { x: categorical.0, y: numeric.0 }
|
|
100
|
+
why: "A violin plot shows the shape of each category's distribution, including modes."
|
|
101
|
+
|
|
102
|
+
# ---- time series --------------------------------------------------------
|
|
103
|
+
- name: datetime_numeric
|
|
104
|
+
requires: { datetime: 1, numeric: 1 }
|
|
105
|
+
candidates:
|
|
106
|
+
- chart: line
|
|
107
|
+
base: 0.94
|
|
108
|
+
map: { x: datetime.0, y: numeric.0, color: "?categorical.0" }
|
|
109
|
+
why: "A date column paired with a measure is a time series, which a line chart shows best."
|
|
110
|
+
- chart: area
|
|
111
|
+
base: 0.68
|
|
112
|
+
map: { x: datetime.0, y: numeric.0 }
|
|
113
|
+
why: "An area chart emphasises cumulative magnitude over the same period."
|
|
114
|
+
- chart: scatter
|
|
115
|
+
base: 0.40
|
|
116
|
+
map: { x: datetime.0, y: numeric.0 }
|
|
117
|
+
why: "Plotting points without connecting them suits irregular or sparse events."
|
|
118
|
+
|
|
119
|
+
# ---- two categorical columns -------------------------------------------
|
|
120
|
+
- name: categorical_pair
|
|
121
|
+
requires: { categorical: 2 }
|
|
122
|
+
candidates:
|
|
123
|
+
- chart: grouped_bar
|
|
124
|
+
base: 0.78
|
|
125
|
+
map: { x: categorical.0, color: categorical.1 }
|
|
126
|
+
agg: count
|
|
127
|
+
why: "Two categorical columns compare cleanly as grouped bars."
|
|
128
|
+
- chart: stacked_bar
|
|
129
|
+
base: 0.64
|
|
130
|
+
map: { x: categorical.0, color: categorical.1 }
|
|
131
|
+
agg: count
|
|
132
|
+
options: { stacked: true }
|
|
133
|
+
why: "Stacking shows each group's composition as well as its total."
|
|
134
|
+
- chart: heatmap
|
|
135
|
+
base: 0.60
|
|
136
|
+
map: { x: categorical.0, y: categorical.1 }
|
|
137
|
+
agg: count
|
|
138
|
+
why: "A heatmap handles the cross-tabulation when either column has many values."
|
|
139
|
+
|
|
140
|
+
- name: categorical_pair_measure
|
|
141
|
+
requires: { categorical: 2, numeric: 1 }
|
|
142
|
+
candidates:
|
|
143
|
+
- chart: grouped_bar
|
|
144
|
+
base: 0.82
|
|
145
|
+
map: { x: categorical.0, y: numeric.0, color: categorical.1 }
|
|
146
|
+
agg: mean
|
|
147
|
+
why: "A measure split across two categorical dimensions reads well as grouped bars."
|
|
148
|
+
- chart: heatmap
|
|
149
|
+
base: 0.70
|
|
150
|
+
map: { x: categorical.0, y: categorical.1, color: numeric.0 }
|
|
151
|
+
agg: mean
|
|
152
|
+
why: "A heatmap shows the measure across every combination of the two dimensions."
|
|
153
|
+
|
|
154
|
+
# ---- many numeric columns ----------------------------------------------
|
|
155
|
+
- name: many_numeric
|
|
156
|
+
requires: { numeric: 3 }
|
|
157
|
+
candidates:
|
|
158
|
+
- chart: correlation_heatmap
|
|
159
|
+
base: 0.76
|
|
160
|
+
map: {}
|
|
161
|
+
why: "With several numeric columns, a correlation matrix surfaces the relationships worth plotting."
|
|
162
|
+
- chart: pair_plot
|
|
163
|
+
base: 0.50
|
|
164
|
+
map: {}
|
|
165
|
+
why: "A pair plot shows every pairwise relationship at once, at the cost of density."
|
|
166
|
+
|
|
167
|
+
# ---- boolean ------------------------------------------------------------
|
|
168
|
+
- name: boolean_split
|
|
169
|
+
requires: { boolean: 1, numeric: 1 }
|
|
170
|
+
candidates:
|
|
171
|
+
- chart: box
|
|
172
|
+
base: 0.72
|
|
173
|
+
map: { x: boolean.0, y: numeric.0 }
|
|
174
|
+
why: "A boolean flag splits the measure into two groups worth comparing side by side."
|
|
175
|
+
- chart: bar
|
|
176
|
+
base: 0.60
|
|
177
|
+
map: { x: boolean.0, y: numeric.0 }
|
|
178
|
+
agg: mean
|
|
179
|
+
why: "Comparing the average on each side of a true/false split."
|
|
180
|
+
|
|
181
|
+
# ---- fallback -----------------------------------------------------------
|
|
182
|
+
- name: text_only
|
|
183
|
+
requires: { text: 1 }
|
|
184
|
+
candidates:
|
|
185
|
+
- chart: bar
|
|
186
|
+
base: 0.25
|
|
187
|
+
map: { x: text.0 }
|
|
188
|
+
agg: count
|
|
189
|
+
options: { top_n: 20 }
|
|
190
|
+
why: "Only free-text columns are available, so the most frequent values are shown as counts."
|