stagewatch 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stagewatch/__init__.py +11 -0
- stagewatch/__main__.py +3 -0
- stagewatch/cli.py +130 -0
- stagewatch/history.py +93 -0
- stagewatch/inspection.py +355 -0
- stagewatch/integrations/__init__.py +1 -0
- stagewatch/integrations/numpy.py +13 -0
- stagewatch/integrations/pandas.py +13 -0
- stagewatch/integrations/torch.py +13 -0
- stagewatch/models.py +117 -0
- stagewatch/pipeline.py +1016 -0
- stagewatch/py.typed +0 -0
- stagewatch/reporters/__init__.py +6 -0
- stagewatch/reporters/html_report.py +63 -0
- stagewatch/reporters/json_report.py +9 -0
- stagewatch/reporters/terminal.py +38 -0
- stagewatch-0.2.0.dist-info/METADATA +373 -0
- stagewatch-0.2.0.dist-info/RECORD +22 -0
- stagewatch-0.2.0.dist-info/WHEEL +5 -0
- stagewatch-0.2.0.dist-info/entry_points.txt +2 -0
- stagewatch-0.2.0.dist-info/licenses/LICENSE +21 -0
- stagewatch-0.2.0.dist-info/top_level.txt +1 -0
stagewatch/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""StageWatch: local-first observability for Python data pipelines."""
|
|
2
|
+
from .inspection import inspect_value, register_inspector
|
|
3
|
+
from .models import Comparison, Finding, RunRecord, StageRecord
|
|
4
|
+
from .pipeline import Pipeline, StageContext, stage
|
|
5
|
+
|
|
6
|
+
__version__ = "0.2.0"
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"Pipeline", "StageContext", "stage", "inspect_value", "register_inspector",
|
|
10
|
+
"Comparison", "Finding", "RunRecord", "StageRecord", "__version__",
|
|
11
|
+
]
|
stagewatch/__main__.py
ADDED
stagewatch/cli.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Command line interface for local StageWatch history databases."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
from .history import HistoryStore
|
|
10
|
+
from .reporters import render_html, render_json, render_terminal
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _default_db(value: str | None) -> Path:
|
|
14
|
+
if value:
|
|
15
|
+
return Path(value)
|
|
16
|
+
folder = Path(".stagewatch")
|
|
17
|
+
databases = sorted(folder.glob("*.sqlite3")) if folder.exists() else []
|
|
18
|
+
if len(databases) == 1:
|
|
19
|
+
return databases[0]
|
|
20
|
+
if not databases:
|
|
21
|
+
raise FileNotFoundError("No StageWatch database found. Pass --db PATH or run an instrumented pipeline first.")
|
|
22
|
+
raise RuntimeError("Multiple StageWatch databases found. Specify one with --db PATH.")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
26
|
+
parser = argparse.ArgumentParser(prog="stagewatch", description="Inspect local Python pipeline execution history.")
|
|
27
|
+
parser.add_argument("--version", action="version", version=f"StageWatch {__version__}")
|
|
28
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
29
|
+
report = sub.add_parser("report", help="Render a recorded pipeline run")
|
|
30
|
+
report.add_argument("--db", help="SQLite history database path")
|
|
31
|
+
report.add_argument("--run-id", help="Run ID (defaults to latest)")
|
|
32
|
+
report.add_argument("--format", choices=("terminal", "json", "html"), default="terminal")
|
|
33
|
+
report.add_argument("--output", help="Write report to this path")
|
|
34
|
+
report.add_argument("--pipeline", help="Filter by pipeline name")
|
|
35
|
+
history = sub.add_parser("history", help="List recorded runs")
|
|
36
|
+
history.add_argument("--db", help="SQLite history database path")
|
|
37
|
+
history.add_argument("--pipeline", help="Filter by pipeline name")
|
|
38
|
+
history.add_argument("--limit", type=int, default=20)
|
|
39
|
+
compare = sub.add_parser("compare", help="Compare two recorded runs")
|
|
40
|
+
compare.add_argument("run_a")
|
|
41
|
+
compare.add_argument("run_b")
|
|
42
|
+
compare.add_argument("--db", help="SQLite history database path")
|
|
43
|
+
compare.add_argument("--pipeline", help="Filter by pipeline name")
|
|
44
|
+
return parser
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def main(argv: list[str] | None = None) -> int:
|
|
48
|
+
# On Windows, the active console code page may not encode all Unicode data
|
|
49
|
+
# (for example, user-defined stage names). Escape unsupported characters
|
|
50
|
+
# rather than letting a report crash with UnicodeEncodeError.
|
|
51
|
+
for stream in (sys.stdout, sys.stderr):
|
|
52
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
53
|
+
if callable(reconfigure):
|
|
54
|
+
try:
|
|
55
|
+
reconfigure(errors="backslashreplace")
|
|
56
|
+
except (OSError, ValueError):
|
|
57
|
+
pass
|
|
58
|
+
parser = build_parser()
|
|
59
|
+
args = parser.parse_args(argv)
|
|
60
|
+
try:
|
|
61
|
+
db = _default_db(args.db)
|
|
62
|
+
store = HistoryStore(db)
|
|
63
|
+
if args.command == "history":
|
|
64
|
+
runs = store.list(args.pipeline, args.limit)
|
|
65
|
+
if not runs:
|
|
66
|
+
print("No runs found.")
|
|
67
|
+
return 0
|
|
68
|
+
print(f"{'RUN ID':<14} {'PIPELINE':<24} {'STATUS':<10} {'DURATION':>14} STARTED")
|
|
69
|
+
for run in runs:
|
|
70
|
+
print(f"{run.run_id:<14} {run.pipeline_name[:23]:<24} {run.status:<10} {run.duration_ms:>10.2f} ms {run.started_at}")
|
|
71
|
+
return 0
|
|
72
|
+
if args.command == "report":
|
|
73
|
+
run = store.get(args.run_id) if args.run_id else store.latest(args.pipeline)
|
|
74
|
+
if run is None or (args.pipeline and run.pipeline_name != args.pipeline):
|
|
75
|
+
print("Requested run was not found.", file=sys.stderr)
|
|
76
|
+
return 2
|
|
77
|
+
if args.format == "terminal":
|
|
78
|
+
content = render_terminal(run)
|
|
79
|
+
elif args.format == "json":
|
|
80
|
+
content = render_json(run)
|
|
81
|
+
else:
|
|
82
|
+
content = render_html(run, store.list(run.pipeline_name, 20))
|
|
83
|
+
output = Path(args.output or f"{run.pipeline_name.lower().replace(' ', '-')}-report.html")
|
|
84
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
output.write_text(content, encoding="utf-8")
|
|
86
|
+
print(f"HTML report written to {output.resolve()}")
|
|
87
|
+
return 0
|
|
88
|
+
if args.output:
|
|
89
|
+
output = Path(args.output)
|
|
90
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
output.write_text(content, encoding="utf-8")
|
|
92
|
+
print(f"Report written to {output.resolve()}")
|
|
93
|
+
else:
|
|
94
|
+
print(content)
|
|
95
|
+
return 0
|
|
96
|
+
if args.command == "compare":
|
|
97
|
+
a, b = store.get(args.run_a), store.get(args.run_b)
|
|
98
|
+
if not a or not b:
|
|
99
|
+
print("One or both run IDs were not found.", file=sys.stderr)
|
|
100
|
+
return 2
|
|
101
|
+
if args.pipeline and (a.pipeline_name != args.pipeline or b.pipeline_name != args.pipeline):
|
|
102
|
+
print("Run does not belong to the requested pipeline.", file=sys.stderr)
|
|
103
|
+
return 2
|
|
104
|
+
# Reuse Pipeline's comparison algorithm without creating a new history database.
|
|
105
|
+
from .pipeline import Pipeline
|
|
106
|
+
pipeline = Pipeline(a.pipeline_name, history=False, storage_dir=None)
|
|
107
|
+
class _Store:
|
|
108
|
+
def latest(self, _name):
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
def list(self, pipeline_name=None, limit=1000):
|
|
112
|
+
return [b, a]
|
|
113
|
+
|
|
114
|
+
def get(self, run_id):
|
|
115
|
+
if run_id == a.run_id:
|
|
116
|
+
return a
|
|
117
|
+
if run_id == b.run_id:
|
|
118
|
+
return b
|
|
119
|
+
return None
|
|
120
|
+
pipeline._store = _Store()
|
|
121
|
+
print(pipeline.compare(a.run_id, b.run_id).explain())
|
|
122
|
+
return 0
|
|
123
|
+
except (OSError, RuntimeError, ValueError, KeyError) as exc:
|
|
124
|
+
print(f"stagewatch: {exc}", file=sys.stderr)
|
|
125
|
+
return 2
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
if __name__ == "__main__":
|
|
130
|
+
raise SystemExit(main())
|
stagewatch/history.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Local SQLite history storage."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import sqlite3
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .models import RunRecord
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class HistoryStore:
|
|
14
|
+
def __init__(self, path: str | Path, max_history: int = 1000) -> None:
|
|
15
|
+
self.path = Path(path).expanduser().resolve()
|
|
16
|
+
self.max_history = max(1, int(max_history))
|
|
17
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
18
|
+
with self._session() as connection:
|
|
19
|
+
connection.execute("""CREATE TABLE IF NOT EXISTS runs (
|
|
20
|
+
run_id TEXT PRIMARY KEY,
|
|
21
|
+
pipeline_name TEXT NOT NULL,
|
|
22
|
+
started_at TEXT NOT NULL,
|
|
23
|
+
payload TEXT NOT NULL
|
|
24
|
+
)""")
|
|
25
|
+
connection.execute("CREATE INDEX IF NOT EXISTS idx_runs_pipeline_started ON runs(pipeline_name, started_at DESC)")
|
|
26
|
+
|
|
27
|
+
def _connect(self) -> sqlite3.Connection:
|
|
28
|
+
"""Create a new SQLite connection.
|
|
29
|
+
|
|
30
|
+
Callers should use :meth:`_session`, which always closes the connection.
|
|
31
|
+
"""
|
|
32
|
+
connection = sqlite3.connect(str(self.path), timeout=10)
|
|
33
|
+
connection.row_factory = sqlite3.Row
|
|
34
|
+
connection.execute("PRAGMA busy_timeout=10000")
|
|
35
|
+
return connection
|
|
36
|
+
|
|
37
|
+
@contextmanager
|
|
38
|
+
def _session(self) -> Iterator[sqlite3.Connection]:
|
|
39
|
+
"""Commit/rollback a database operation and ALWAYS close its connection.
|
|
40
|
+
|
|
41
|
+
The sqlite3 connection context manager alone does not close connections.
|
|
42
|
+
Explicit closure is essential on Windows, where open handles prevent
|
|
43
|
+
temporary directories and SQLite files from being removed.
|
|
44
|
+
"""
|
|
45
|
+
connection = self._connect()
|
|
46
|
+
try:
|
|
47
|
+
with connection:
|
|
48
|
+
yield connection
|
|
49
|
+
finally:
|
|
50
|
+
connection.close()
|
|
51
|
+
|
|
52
|
+
def save(self, run: RunRecord) -> None:
|
|
53
|
+
payload = json.dumps(run.to_dict(), ensure_ascii=False, separators=(",", ":"))
|
|
54
|
+
with self._session() as connection:
|
|
55
|
+
connection.execute(
|
|
56
|
+
"INSERT OR REPLACE INTO runs(run_id, pipeline_name, started_at, payload) VALUES (?, ?, ?, ?)",
|
|
57
|
+
(run.run_id, run.pipeline_name, run.started_at, payload),
|
|
58
|
+
)
|
|
59
|
+
connection.execute(
|
|
60
|
+
"DELETE FROM runs WHERE run_id IN (SELECT run_id FROM runs WHERE pipeline_name=? ORDER BY started_at DESC LIMIT -1 OFFSET ?)",
|
|
61
|
+
(run.pipeline_name, self.max_history),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def get(self, run_id: str) -> RunRecord | None:
|
|
65
|
+
with self._session() as connection:
|
|
66
|
+
row = connection.execute("SELECT payload FROM runs WHERE run_id=?", (run_id,)).fetchone()
|
|
67
|
+
return RunRecord.from_dict(json.loads(row["payload"])) if row else None
|
|
68
|
+
|
|
69
|
+
def list(self, pipeline_name: str | None = None, limit: int = 20) -> list[RunRecord]:
|
|
70
|
+
limit = max(1, min(int(limit), 10000))
|
|
71
|
+
with self._session() as connection:
|
|
72
|
+
if pipeline_name:
|
|
73
|
+
rows = connection.execute("SELECT payload FROM runs WHERE pipeline_name=? ORDER BY started_at DESC LIMIT ?", (pipeline_name, limit)).fetchall()
|
|
74
|
+
else:
|
|
75
|
+
rows = connection.execute("SELECT payload FROM runs ORDER BY started_at DESC LIMIT ?", (limit,)).fetchall()
|
|
76
|
+
return [RunRecord.from_dict(json.loads(row["payload"])) for row in rows]
|
|
77
|
+
|
|
78
|
+
def latest(self, pipeline_name: str | None = None) -> RunRecord | None:
|
|
79
|
+
items = self.list(pipeline_name=pipeline_name, limit=1)
|
|
80
|
+
return items[0] if items else None
|
|
81
|
+
|
|
82
|
+
def delete_all(self, pipeline_name: str | None = None) -> int:
|
|
83
|
+
with self._session() as connection:
|
|
84
|
+
if pipeline_name:
|
|
85
|
+
cursor = connection.execute("DELETE FROM runs WHERE pipeline_name=?", (pipeline_name,))
|
|
86
|
+
else:
|
|
87
|
+
cursor = connection.execute("DELETE FROM runs")
|
|
88
|
+
return cursor.rowcount
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def load_history(path: str | Path) -> HistoryStore:
|
|
92
|
+
"""Open an existing StageWatch database (or initialize an empty one)."""
|
|
93
|
+
return HistoryStore(path)
|
stagewatch/inspection.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Safe, bounded metadata inspection without mandatory third-party dependencies."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
import statistics
|
|
6
|
+
import sys
|
|
7
|
+
from collections.abc import Callable, Mapping
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
_INSPECTORS: list[tuple[Callable[[Any], bool], Callable[[Any, int], dict[str, Any]]]] = []
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register_inspector(predicate: Callable[[Any], bool], inspector: Callable[[Any, int], dict[str, Any]]) -> None:
|
|
14
|
+
"""Register a custom inspector. New inspectors are checked before built-ins."""
|
|
15
|
+
if not callable(predicate) or not callable(inspector):
|
|
16
|
+
raise TypeError("predicate and inspector must be callable")
|
|
17
|
+
_INSPECTORS.insert(0, (predicate, inspector))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _safe_int(value: Any) -> int | None:
|
|
21
|
+
try:
|
|
22
|
+
return int(value)
|
|
23
|
+
except Exception:
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _deep_size(value: Any, seen: set[int] | None = None, depth: int = 0) -> int:
|
|
28
|
+
if seen is None:
|
|
29
|
+
seen = set()
|
|
30
|
+
ident = id(value)
|
|
31
|
+
if ident in seen or depth > 5:
|
|
32
|
+
return 0
|
|
33
|
+
seen.add(ident)
|
|
34
|
+
size = sys.getsizeof(value, 0)
|
|
35
|
+
if isinstance(value, Mapping):
|
|
36
|
+
for key, item in list(value.items())[:1000]:
|
|
37
|
+
size += _deep_size(key, seen, depth + 1) + _deep_size(item, seen, depth + 1)
|
|
38
|
+
elif isinstance(value, (list, tuple, set, frozenset)):
|
|
39
|
+
for item in list(value)[:1000]:
|
|
40
|
+
size += _deep_size(item, seen, depth + 1)
|
|
41
|
+
return size
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _numeric_summary(numbers: list[float], multiplier: float = 1.5) -> dict[str, Any]:
|
|
45
|
+
"""Summarize finite values and Tukey-IQR outliers; never return raw values."""
|
|
46
|
+
if not numbers:
|
|
47
|
+
return {"numeric_count": 0, "outlier_count": 0, "outlier_ratio": 0.0}
|
|
48
|
+
ordered = sorted(numbers)
|
|
49
|
+
count = len(ordered)
|
|
50
|
+
|
|
51
|
+
def percentile(fraction: float) -> float:
|
|
52
|
+
if count == 1:
|
|
53
|
+
return ordered[0]
|
|
54
|
+
position = (count - 1) * fraction
|
|
55
|
+
lower = int(position)
|
|
56
|
+
upper = min(lower + 1, count - 1)
|
|
57
|
+
weight = position - lower
|
|
58
|
+
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
|
|
59
|
+
|
|
60
|
+
q1 = percentile(0.25)
|
|
61
|
+
median = percentile(0.5)
|
|
62
|
+
q3 = percentile(0.75)
|
|
63
|
+
iqr = q3 - q1
|
|
64
|
+
low_fence = q1 - multiplier * iqr
|
|
65
|
+
high_fence = q3 + multiplier * iqr
|
|
66
|
+
outliers = sum(value < low_fence or value > high_fence for value in ordered) if iqr > 0 else sum(value != median for value in ordered)
|
|
67
|
+
mean = math.fsum(ordered) / count
|
|
68
|
+
std = statistics.pstdev(ordered) if count > 1 else 0.0
|
|
69
|
+
return {
|
|
70
|
+
"numeric_count": count,
|
|
71
|
+
"minimum": ordered[0],
|
|
72
|
+
"maximum": ordered[-1],
|
|
73
|
+
"mean": mean,
|
|
74
|
+
"std": std,
|
|
75
|
+
"median": median,
|
|
76
|
+
"q1": q1,
|
|
77
|
+
"q3": q3,
|
|
78
|
+
"iqr": iqr,
|
|
79
|
+
"outlier_count": outliers,
|
|
80
|
+
"outlier_ratio": outliers / count,
|
|
81
|
+
"outlier_lower_fence": low_fence,
|
|
82
|
+
"outlier_upper_fence": high_fence,
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _infer_builtin_shape(value: Any, *, max_depth: int = 8, max_items: int = 10000) -> list[int] | None:
|
|
87
|
+
"""Infer a rectangular nested list/tuple shape without copying its payload."""
|
|
88
|
+
if not isinstance(value, (list, tuple)):
|
|
89
|
+
return None
|
|
90
|
+
current = [value]
|
|
91
|
+
shape: list[int] = []
|
|
92
|
+
inspected = 0
|
|
93
|
+
for _ in range(max_depth):
|
|
94
|
+
lengths = []
|
|
95
|
+
children: list[Any] = []
|
|
96
|
+
sequence_flags: list[bool] = []
|
|
97
|
+
for node in current:
|
|
98
|
+
if isinstance(node, (list, tuple)):
|
|
99
|
+
lengths.append(len(node))
|
|
100
|
+
inspected += len(node)
|
|
101
|
+
if inspected > max_items:
|
|
102
|
+
return None
|
|
103
|
+
children.extend(node)
|
|
104
|
+
sequence_flags.extend(isinstance(item, (list, tuple)) for item in node)
|
|
105
|
+
else:
|
|
106
|
+
return shape or None
|
|
107
|
+
if not lengths or len(set(lengths)) != 1:
|
|
108
|
+
return None
|
|
109
|
+
shape.append(lengths[0])
|
|
110
|
+
if not children or not any(sequence_flags):
|
|
111
|
+
return shape
|
|
112
|
+
if not all(sequence_flags):
|
|
113
|
+
return None
|
|
114
|
+
current = children
|
|
115
|
+
return shape
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _walk_builtin(value: Any, limit: int = 10000, outlier_iqr_multiplier: float = 1.5) -> dict[str, Any]:
|
|
119
|
+
numeric: list[float] = []
|
|
120
|
+
nan_count = inf_count = none_count = visited = 0
|
|
121
|
+
stack = [(value, 0)]
|
|
122
|
+
while stack and visited < limit:
|
|
123
|
+
item, depth = stack.pop()
|
|
124
|
+
visited += 1
|
|
125
|
+
if item is None:
|
|
126
|
+
none_count += 1
|
|
127
|
+
elif isinstance(item, bool):
|
|
128
|
+
numeric.append(float(item))
|
|
129
|
+
elif isinstance(item, (int, float)):
|
|
130
|
+
number = float(item)
|
|
131
|
+
if math.isnan(number):
|
|
132
|
+
nan_count += 1
|
|
133
|
+
elif math.isinf(number):
|
|
134
|
+
inf_count += 1
|
|
135
|
+
else:
|
|
136
|
+
numeric.append(number)
|
|
137
|
+
elif depth < 5 and isinstance(item, Mapping):
|
|
138
|
+
stack.extend((v, depth + 1) for v in list(item.values())[:limit])
|
|
139
|
+
elif depth < 5 and isinstance(item, (list, tuple, set, frozenset)):
|
|
140
|
+
stack.extend((v, depth + 1) for v in list(item)[:limit])
|
|
141
|
+
result: dict[str, Any] = {
|
|
142
|
+
"nan_count": nan_count,
|
|
143
|
+
"inf_count": inf_count,
|
|
144
|
+
"none_count": none_count,
|
|
145
|
+
"scan_truncated": bool(stack),
|
|
146
|
+
}
|
|
147
|
+
result.update(_numeric_summary(numeric, outlier_iqr_multiplier))
|
|
148
|
+
return result
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _inspect_numpy(value: Any, deep_scan_limit_bytes: int, outlier_iqr_multiplier: float) -> dict[str, Any]:
|
|
152
|
+
import numpy as np
|
|
153
|
+
|
|
154
|
+
result: dict[str, Any] = {
|
|
155
|
+
"shape": [int(x) for x in value.shape],
|
|
156
|
+
"dtype": str(value.dtype),
|
|
157
|
+
"estimated_size_bytes": int(value.nbytes),
|
|
158
|
+
"length": int(value.size),
|
|
159
|
+
"empty": bool(value.size == 0),
|
|
160
|
+
}
|
|
161
|
+
if value.dtype.kind in "biuf" and value.nbytes <= deep_scan_limit_bytes:
|
|
162
|
+
try:
|
|
163
|
+
result["nan_count"] = int(np.isnan(value).sum())
|
|
164
|
+
result["inf_count"] = int(np.isinf(value).sum())
|
|
165
|
+
finite = value[np.isfinite(value)]
|
|
166
|
+
result.update(_numeric_summary([float(x) for x in finite.ravel().tolist()], outlier_iqr_multiplier))
|
|
167
|
+
except (TypeError, ValueError, FloatingPointError, OverflowError):
|
|
168
|
+
result["scan_skipped"] = "numeric summary could not be computed"
|
|
169
|
+
else:
|
|
170
|
+
result["scan_skipped"] = "array exceeds scan limit or is non-real numeric"
|
|
171
|
+
return result
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _inspect_pandas(value: Any, deep_scan_limit_bytes: int, outlier_iqr_multiplier: float) -> dict[str, Any]:
|
|
175
|
+
import numpy as np
|
|
176
|
+
import pandas as pd
|
|
177
|
+
|
|
178
|
+
shape = [int(x) for x in value.shape]
|
|
179
|
+
memory_value = value.memory_usage(index=True, deep=False)
|
|
180
|
+
memory = int(memory_value.sum() if hasattr(memory_value, "sum") else memory_value)
|
|
181
|
+
dtype_value = getattr(value, "dtypes", None)
|
|
182
|
+
dtype_summary = (
|
|
183
|
+
[str(x) for x in dtype_value]
|
|
184
|
+
if dtype_value is not None and not isinstance(dtype_value, str) and hasattr(dtype_value, "__iter__")
|
|
185
|
+
else str(getattr(value, "dtype", dtype_value))
|
|
186
|
+
)
|
|
187
|
+
result: dict[str, Any] = {
|
|
188
|
+
"shape": shape,
|
|
189
|
+
"dtype": dtype_summary,
|
|
190
|
+
"estimated_size_bytes": memory,
|
|
191
|
+
"length": len(value),
|
|
192
|
+
"empty": bool(value.empty),
|
|
193
|
+
}
|
|
194
|
+
if memory <= deep_scan_limit_bytes:
|
|
195
|
+
try:
|
|
196
|
+
missing = value.isna().sum()
|
|
197
|
+
result["missing_count"] = int(missing.sum() if hasattr(missing, "sum") else missing)
|
|
198
|
+
if isinstance(value, pd.DataFrame):
|
|
199
|
+
numeric = value.select_dtypes(include=["number"])
|
|
200
|
+
series_items = [(str(column)[:100], numeric[column]) for column in numeric.columns]
|
|
201
|
+
elif pd.api.types.is_numeric_dtype(value.dtype):
|
|
202
|
+
series_items = [(str(value.name or "value")[:100], value)]
|
|
203
|
+
else:
|
|
204
|
+
series_items = []
|
|
205
|
+
|
|
206
|
+
column_summaries: dict[str, Any] = {}
|
|
207
|
+
all_finite: list[float] = []
|
|
208
|
+
all_nan = all_inf = outlier_count = 0
|
|
209
|
+
for column_name, series in series_items:
|
|
210
|
+
array = series.to_numpy(dtype=float, na_value=float("nan"), copy=False).ravel()
|
|
211
|
+
nan_count = int(np.isnan(array).sum())
|
|
212
|
+
inf_count = int(np.isinf(array).sum())
|
|
213
|
+
finite = array[np.isfinite(array)]
|
|
214
|
+
column_summary = _numeric_summary(finite.tolist(), outlier_iqr_multiplier)
|
|
215
|
+
column_summary["nan_count"] = nan_count
|
|
216
|
+
column_summary["inf_count"] = inf_count
|
|
217
|
+
column_summaries[column_name] = column_summary
|
|
218
|
+
all_finite.extend(float(item) for item in finite.tolist())
|
|
219
|
+
all_nan += nan_count
|
|
220
|
+
all_inf += inf_count
|
|
221
|
+
outlier_count += int(column_summary.get("outlier_count", 0))
|
|
222
|
+
|
|
223
|
+
result["nan_count"] = all_nan
|
|
224
|
+
result["inf_count"] = all_inf
|
|
225
|
+
result["numeric_columns"] = column_summaries
|
|
226
|
+
result.update(_numeric_summary(all_finite, outlier_iqr_multiplier))
|
|
227
|
+
# For tabular data, count outliers within each numeric column rather
|
|
228
|
+
# than flattening unrelated columns onto one common value scale.
|
|
229
|
+
result["outlier_count"] = outlier_count
|
|
230
|
+
result["outlier_ratio"] = outlier_count / max(1, sum(int(item["numeric_count"]) for item in column_summaries.values()))
|
|
231
|
+
except (TypeError, ValueError, OverflowError):
|
|
232
|
+
result["scan_skipped"] = "tabular numeric summary could not be computed"
|
|
233
|
+
else:
|
|
234
|
+
result["scan_skipped"] = "tabular data exceeds scan limit"
|
|
235
|
+
return result
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _inspect_torch(value: Any, deep_scan_limit_bytes: int, inspect_gpu: bool, outlier_iqr_multiplier: float) -> dict[str, Any]:
|
|
239
|
+
result: dict[str, Any] = {
|
|
240
|
+
"shape": [int(x) for x in value.shape],
|
|
241
|
+
"dtype": str(value.dtype),
|
|
242
|
+
"device": str(value.device),
|
|
243
|
+
"length": int(value.numel()),
|
|
244
|
+
"estimated_size_bytes": int(value.numel() * value.element_size()),
|
|
245
|
+
"empty": bool(value.numel() == 0),
|
|
246
|
+
}
|
|
247
|
+
should_scan = result["estimated_size_bytes"] <= deep_scan_limit_bytes and (str(value.device) == "cpu" or inspect_gpu)
|
|
248
|
+
if should_scan and (getattr(value, "is_floating_point", lambda: False)() or getattr(value, "is_complex", lambda: False)()):
|
|
249
|
+
try:
|
|
250
|
+
detached = value.detach()
|
|
251
|
+
result["nan_count"] = int(detached.isnan().sum().item())
|
|
252
|
+
result["inf_count"] = int(detached.isinf().sum().item())
|
|
253
|
+
finite = detached[detached.isfinite()]
|
|
254
|
+
if finite.numel() and not getattr(value, "is_complex", lambda: False)():
|
|
255
|
+
result.update(_numeric_summary([float(x) for x in finite.float().reshape(-1).tolist()], outlier_iqr_multiplier))
|
|
256
|
+
except Exception:
|
|
257
|
+
result["scan_skipped"] = "tensor scan was unavailable"
|
|
258
|
+
elif result["estimated_size_bytes"] > deep_scan_limit_bytes:
|
|
259
|
+
result["scan_skipped"] = "tensor exceeds scan limit"
|
|
260
|
+
elif str(value.device) != "cpu":
|
|
261
|
+
result["scan_skipped"] = "GPU scanning is disabled by default"
|
|
262
|
+
return result
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def inspect_value(
|
|
266
|
+
value: Any, *, deep_scan_limit_bytes: int = 1_000_000, inspect_gpu: bool = False,
|
|
267
|
+
outlier_iqr_multiplier: float = 1.5,
|
|
268
|
+
) -> dict[str, Any]:
|
|
269
|
+
"""Return metadata and safe numerical summaries; never return raw user data."""
|
|
270
|
+
if deep_scan_limit_bytes < 0:
|
|
271
|
+
raise ValueError("deep_scan_limit_bytes must be >= 0")
|
|
272
|
+
if outlier_iqr_multiplier <= 0:
|
|
273
|
+
raise ValueError("outlier_iqr_multiplier must be > 0")
|
|
274
|
+
result: dict[str, Any] = {"type": f"{type(value).__module__}.{type(value).__qualname__}"}
|
|
275
|
+
if value is None:
|
|
276
|
+
result.update({"estimated_size_bytes": sys.getsizeof(None), "length": None, "empty": True, "none_count": 1})
|
|
277
|
+
elif isinstance(value, (str, bytes, bytearray)):
|
|
278
|
+
result.update({"length": len(value), "empty": len(value) == 0, "estimated_size_bytes": sys.getsizeof(value)})
|
|
279
|
+
elif isinstance(value, (bool, int, float)):
|
|
280
|
+
result.update({"length": None, "empty": False, "estimated_size_bytes": sys.getsizeof(value)})
|
|
281
|
+
result.update(_walk_builtin(value, outlier_iqr_multiplier=outlier_iqr_multiplier))
|
|
282
|
+
elif isinstance(value, Mapping):
|
|
283
|
+
keys = list(value.keys())
|
|
284
|
+
result.update({
|
|
285
|
+
"length": len(value), "empty": len(value) == 0,
|
|
286
|
+
"estimated_size_bytes": _deep_size(value),
|
|
287
|
+
"mapping_keys": [str(key)[:100] for key in keys[:50]],
|
|
288
|
+
"mapping_key_count": len(keys),
|
|
289
|
+
})
|
|
290
|
+
walked = _walk_builtin(value, outlier_iqr_multiplier=outlier_iqr_multiplier)
|
|
291
|
+
result.update(walked)
|
|
292
|
+
# Numeric dictionary fields often have unrelated units (e.g. image
|
|
293
|
+
# width, channel count and luminance). Never apply one IQR fence to
|
|
294
|
+
# values from different keys. Analyse numeric vectors per field.
|
|
295
|
+
numeric_fields: dict[str, Any] = {}
|
|
296
|
+
for key, item in list(value.items())[:100]:
|
|
297
|
+
if isinstance(item, (list, tuple)) and item and all(
|
|
298
|
+
element is None or isinstance(element, (bool, int, float)) for element in item[:10000]
|
|
299
|
+
):
|
|
300
|
+
field_summary = _walk_builtin(item, outlier_iqr_multiplier=outlier_iqr_multiplier)
|
|
301
|
+
if int(field_summary.get("numeric_count", 0)) >= 2:
|
|
302
|
+
numeric_fields[str(key)[:100]] = field_summary
|
|
303
|
+
result["numeric_fields"] = numeric_fields
|
|
304
|
+
if numeric_fields:
|
|
305
|
+
numeric_total = sum(int(item.get("numeric_count", 0)) for item in numeric_fields.values())
|
|
306
|
+
outlier_total = sum(int(item.get("outlier_count", 0)) for item in numeric_fields.values())
|
|
307
|
+
result["outlier_count"] = outlier_total
|
|
308
|
+
result["outlier_ratio"] = outlier_total / max(1, numeric_total)
|
|
309
|
+
else:
|
|
310
|
+
# Scalars from distinct keys may not share a meaningful scale.
|
|
311
|
+
result["outlier_count"] = 0
|
|
312
|
+
result["outlier_ratio"] = 0.0
|
|
313
|
+
elif isinstance(value, (list, tuple, set, frozenset)):
|
|
314
|
+
result.update({"length": len(value), "empty": len(value) == 0, "estimated_size_bytes": _deep_size(value)})
|
|
315
|
+
if isinstance(value, (list, tuple)):
|
|
316
|
+
inferred_shape = _infer_builtin_shape(value)
|
|
317
|
+
if inferred_shape is not None:
|
|
318
|
+
result["shape"] = inferred_shape
|
|
319
|
+
result.update(_walk_builtin(value, outlier_iqr_multiplier=outlier_iqr_multiplier))
|
|
320
|
+
elif type(value).__module__.startswith("numpy") and hasattr(value, "shape") and hasattr(value, "nbytes"):
|
|
321
|
+
result.update(_inspect_numpy(value, deep_scan_limit_bytes, outlier_iqr_multiplier))
|
|
322
|
+
elif type(value).__module__.startswith("pandas") and hasattr(value, "memory_usage"):
|
|
323
|
+
result.update(_inspect_pandas(value, deep_scan_limit_bytes, outlier_iqr_multiplier))
|
|
324
|
+
elif type(value).__module__.startswith("torch") and hasattr(value, "numel") and hasattr(value, "element_size"):
|
|
325
|
+
result.update(_inspect_torch(value, deep_scan_limit_bytes, inspect_gpu, outlier_iqr_multiplier))
|
|
326
|
+
else:
|
|
327
|
+
for predicate, inspector in tuple(_INSPECTORS):
|
|
328
|
+
try:
|
|
329
|
+
if predicate(value):
|
|
330
|
+
result.update(inspector(value, deep_scan_limit_bytes))
|
|
331
|
+
break
|
|
332
|
+
except Exception as exc:
|
|
333
|
+
result["custom_inspector_error"] = type(exc).__name__
|
|
334
|
+
else:
|
|
335
|
+
shape = getattr(value, "shape", None)
|
|
336
|
+
if shape is not None:
|
|
337
|
+
try:
|
|
338
|
+
result["shape"] = [int(x) for x in shape]
|
|
339
|
+
except Exception:
|
|
340
|
+
result["shape"] = repr(shape)
|
|
341
|
+
dtype = getattr(value, "dtype", None)
|
|
342
|
+
if dtype is not None:
|
|
343
|
+
result["dtype"] = str(dtype)
|
|
344
|
+
nbytes = _safe_int(getattr(value, "nbytes", None))
|
|
345
|
+
result["estimated_size_bytes"] = nbytes if nbytes is not None else _deep_size(value)
|
|
346
|
+
try:
|
|
347
|
+
result["length"] = len(value)
|
|
348
|
+
result["empty"] = len(value) == 0
|
|
349
|
+
except Exception:
|
|
350
|
+
result["length"] = None
|
|
351
|
+
result["empty"] = False
|
|
352
|
+
result.setdefault("estimated_size_bytes", _deep_size(value))
|
|
353
|
+
result.setdefault("length", None)
|
|
354
|
+
result.setdefault("empty", False)
|
|
355
|
+
return result
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional integration helpers. Core StageWatch works without these extras."""
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""NumPy integration helpers (install with ``stagewatch[numpy]``)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from ..inspection import inspect_value
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def inspect_array(value: Any, *, deep_scan_limit_bytes: int = 1_000_000) -> dict[str, Any]:
|
|
10
|
+
import numpy # noqa: F401
|
|
11
|
+
if not isinstance(value, numpy.ndarray):
|
|
12
|
+
raise TypeError("inspect_array expects a numpy.ndarray")
|
|
13
|
+
return inspect_value(value, deep_scan_limit_bytes=deep_scan_limit_bytes)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Pandas integration helpers (install with ``stagewatch[pandas]``)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from ..inspection import inspect_value
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def inspect_frame(value: Any, *, deep_scan_limit_bytes: int = 1_000_000) -> dict[str, Any]:
|
|
10
|
+
import pandas
|
|
11
|
+
if not isinstance(value, (pandas.DataFrame, pandas.Series)):
|
|
12
|
+
raise TypeError("inspect_frame expects a pandas DataFrame or Series")
|
|
13
|
+
return inspect_value(value, deep_scan_limit_bytes=deep_scan_limit_bytes)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""PyTorch integration helpers (install with ``stagewatch[torch]``)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from ..inspection import inspect_value
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def inspect_tensor(value: Any, *, deep_scan_limit_bytes: int = 1_000_000, inspect_gpu: bool = False) -> dict[str, Any]:
|
|
10
|
+
import torch
|
|
11
|
+
if not isinstance(value, torch.Tensor):
|
|
12
|
+
raise TypeError("inspect_tensor expects a torch.Tensor")
|
|
13
|
+
return inspect_value(value, deep_scan_limit_bytes=deep_scan_limit_bytes, inspect_gpu=inspect_gpu)
|