matchlab 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- matchlab/__init__.py +55 -0
- matchlab/adapters/__init__.py +24 -0
- matchlab/adapters/base.py +446 -0
- matchlab/adapters/duckdb.py +659 -0
- matchlab/cli.py +177 -0
- matchlab/core/__init__.py +1 -0
- matchlab/core/dataframes.py +254 -0
- matchlab/core/dsu.py +62 -0
- matchlab/core/exceptions.py +40 -0
- matchlab/core/hash.py +141 -0
- matchlab/core/kinds.py +12 -0
- matchlab/core/logging.py +190 -0
- matchlab/core/resolver_output.py +173 -0
- matchlab/core/schemas.py +82 -0
- matchlab/core/sql.py +12 -0
- matchlab/document.py +329 -0
- matchlab/eval/__init__.py +11 -0
- matchlab/eval/judgements.py +41 -0
- matchlab/eval/metrics.py +244 -0
- matchlab/eval/samples.py +415 -0
- matchlab/eval/tui/__init__.py +66 -0
- matchlab/eval/tui/app.py +418 -0
- matchlab/eval/tui/modals.py +84 -0
- matchlab/eval/tui/styles.tcss +184 -0
- matchlab/eval/tui/widgets/__init__.py +1 -0
- matchlab/eval/tui/widgets/assignment.py +72 -0
- matchlab/eval/tui/widgets/styling.py +75 -0
- matchlab/eval/tui/widgets/table.py +285 -0
- matchlab/lineage.py +239 -0
- matchlab/locations.py +347 -0
- matchlab/models/__init__.py +10 -0
- matchlab/models/comparison.py +67 -0
- matchlab/models/dedupers/__init__.py +5 -0
- matchlab/models/dedupers/base.py +38 -0
- matchlab/models/dedupers/naive.py +77 -0
- matchlab/models/linkers/__init__.py +9 -0
- matchlab/models/linkers/base.py +41 -0
- matchlab/models/linkers/deterministic.py +229 -0
- matchlab/models/linkers/splinklinker.py +262 -0
- matchlab/models/linkers/weighteddeterministic.py +143 -0
- matchlab/models/models.py +198 -0
- matchlab/progress.py +455 -0
- matchlab/recordstep.py +181 -0
- matchlab/resolvers/__init__.py +10 -0
- matchlab/resolvers/base.py +63 -0
- matchlab/resolvers/components.py +66 -0
- matchlab/resolvers/resolvers.py +409 -0
- matchlab/sources.py +291 -0
- matchlab/specs.py +129 -0
- matchlab/steps.py +284 -0
- matchlab/testkit/__init__.py +104 -0
- matchlab/testkit/_generate.py +240 -0
- matchlab/testkit/compare.py +178 -0
- matchlab/testkit/entities.py +290 -0
- matchlab/testkit/features.py +146 -0
- matchlab/testkit/linked.py +527 -0
- matchlab/testkit/matchers.py +233 -0
- matchlab/testkit/models.py +165 -0
- matchlab/testkit/sources.py +327 -0
- matchlab/transformers/__init__.py +24 -0
- matchlab/transformers/base.py +41 -0
- matchlab/transformers/clean.py +48 -0
- matchlab/transformers/explode.py +67 -0
- matchlab/transformers/group.py +56 -0
- matchlab/transformers/select.py +31 -0
- matchlab/transformers/transform.py +101 -0
- matchlab-0.1.0.dev1.dist-info/METADATA +88 -0
- matchlab-0.1.0.dev1.dist-info/RECORD +72 -0
- matchlab-0.1.0.dev1.dist-info/WHEEL +5 -0
- matchlab-0.1.0.dev1.dist-info/entry_points.txt +2 -0
- matchlab-0.1.0.dev1.dist-info/licenses/LICENSE +21 -0
- matchlab-0.1.0.dev1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Model — a deduper or linker producing scored candidate edges."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
5
|
+
|
|
6
|
+
import polars as pl
|
|
7
|
+
|
|
8
|
+
from matchlab.adapters import Adapter, Fingerprint
|
|
9
|
+
from matchlab.core.kinds import StepKind
|
|
10
|
+
from matchlab.core.logging import logger
|
|
11
|
+
from matchlab.core.schemas import SCHEMA_MODEL_EDGES
|
|
12
|
+
from matchlab.models import dedupers, linkers
|
|
13
|
+
from matchlab.models.dedupers.base import Deduper, DeduperSettings
|
|
14
|
+
from matchlab.models.linkers.base import Linker, LinkerSettings
|
|
15
|
+
from matchlab.recordstep import RecordStep
|
|
16
|
+
from matchlab.specs import ModelSpec, ModelType
|
|
17
|
+
from matchlab.steps import Step
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
# `matchlab.resolvers` imports this module
|
|
21
|
+
from matchlab.resolvers import Resolver
|
|
22
|
+
from matchlab.resolvers.base import ResolverMethod, ResolverSettings
|
|
23
|
+
|
|
24
|
+
_MODEL_CLASSES: dict[str, type[Linker] | type[Deduper]] = {
|
|
25
|
+
**dict(inspect.getmembers(dedupers, inspect.isclass)),
|
|
26
|
+
**dict(inspect.getmembers(linkers, inspect.isclass)),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def add_model_class(model_class: type[Linker] | type[Deduper]) -> None:
|
|
31
|
+
"""Register a custom deduper or linker so it can be named in a plan."""
|
|
32
|
+
if not issubclass(model_class, Linker | Deduper):
|
|
33
|
+
raise ValueError("The argument is not a subclass of Deduper or Linker.")
|
|
34
|
+
_MODEL_CLASSES[model_class.__name__] = model_class
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def normalise_model_scores(scores: pl.DataFrame) -> pl.DataFrame:
|
|
38
|
+
"""Validate a methodology's raw output and cast it to `SCHEMA_MODEL_EDGES`.
|
|
39
|
+
|
|
40
|
+
Raises `ValueError` if `scores` isn't a `DataFrame` with exactly `left_id`,
|
|
41
|
+
`right_id`, and a numeric `score` in `[0.0, 1.0]`. An unordered pair can appear
|
|
42
|
+
twice, once as `(a, b)` and once as `(b, a)`. When that happens, this keeps only
|
|
43
|
+
the highest-scoring row and logs a warning, since a resolver expects one edge per
|
|
44
|
+
pair.
|
|
45
|
+
"""
|
|
46
|
+
if not isinstance(scores, pl.DataFrame):
|
|
47
|
+
raise ValueError(f"Expected a polars DataFrame, got {type(scores)}.")
|
|
48
|
+
|
|
49
|
+
expected_fields = set(SCHEMA_MODEL_EDGES.names)
|
|
50
|
+
if set(scores.columns) != expected_fields:
|
|
51
|
+
raise ValueError(f"Expected {expected_fields}.\nFound {set(scores.columns)}.")
|
|
52
|
+
|
|
53
|
+
if scores.height == 0:
|
|
54
|
+
scores = pl.DataFrame(schema=pl.Schema(SCHEMA_MODEL_EDGES))
|
|
55
|
+
|
|
56
|
+
if not scores["score"].dtype.is_numeric():
|
|
57
|
+
raise ValueError(
|
|
58
|
+
"Score column must contain numeric values in the range [0.0, 1.0]."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
normalised_scores = scores.with_columns(pl.col("score").cast(pl.Float32))
|
|
62
|
+
invalid_scores = normalised_scores.filter(
|
|
63
|
+
pl.col("score").is_null()
|
|
64
|
+
| pl.col("score").is_nan()
|
|
65
|
+
| (pl.col("score") < 0.0)
|
|
66
|
+
| (pl.col("score") > 1.0)
|
|
67
|
+
)
|
|
68
|
+
if invalid_scores.height:
|
|
69
|
+
min_score = normalised_scores["score"].min()
|
|
70
|
+
max_score = normalised_scores["score"].max()
|
|
71
|
+
raise ValueError(f"Score range misconfigured: [{min_score}, {max_score}]")
|
|
72
|
+
|
|
73
|
+
unique_scores = (
|
|
74
|
+
normalised_scores.with_columns(
|
|
75
|
+
pl.concat_list(
|
|
76
|
+
[pl.col("left_id").cast(pl.Utf8), pl.col("right_id").cast(pl.Utf8)]
|
|
77
|
+
)
|
|
78
|
+
.list.sort()
|
|
79
|
+
.list.join("_")
|
|
80
|
+
.alias("sorted_ids")
|
|
81
|
+
)
|
|
82
|
+
.sort("score", descending=True) # sort so largest score comes first
|
|
83
|
+
.unique(
|
|
84
|
+
subset=["sorted_ids"], keep="first"
|
|
85
|
+
) # keep first occurrence after sorting
|
|
86
|
+
).drop("sorted_ids")
|
|
87
|
+
if len(scores) != len(unique_scores):
|
|
88
|
+
logger.warning("Duplicate pairs! Keeping only pairs with highest score.")
|
|
89
|
+
|
|
90
|
+
return unique_scores.cast(pl.Schema(SCHEMA_MODEL_EDGES))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class Model(Step):
|
|
94
|
+
"""A step that runs one methodology (a Deduper or a Linker) over record steps."""
|
|
95
|
+
|
|
96
|
+
kind: ClassVar[StepKind] = StepKind.MODEL
|
|
97
|
+
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
left: RecordStep,
|
|
101
|
+
model_class: type[Deduper] | type[Linker] | str,
|
|
102
|
+
model_settings: DeduperSettings | LinkerSettings | dict,
|
|
103
|
+
right: RecordStep | None = None,
|
|
104
|
+
) -> None:
|
|
105
|
+
"""Define a model.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
left: The record step to deduplicate, or the left side of a link.
|
|
109
|
+
model_class: A `Deduper`/`Linker` subclass, or its registered name.
|
|
110
|
+
model_settings: The settings object for that class, or a dict.
|
|
111
|
+
right: The right side of a link. Omit for a deduper.
|
|
112
|
+
"""
|
|
113
|
+
self.left = left
|
|
114
|
+
self.right = right
|
|
115
|
+
|
|
116
|
+
self.model_class = (
|
|
117
|
+
_MODEL_CLASSES[model_class] if isinstance(model_class, str) else model_class
|
|
118
|
+
)
|
|
119
|
+
self.model_instance = self.model_class(settings=model_settings)
|
|
120
|
+
self.model_type = (
|
|
121
|
+
ModelType.LINKER
|
|
122
|
+
if issubclass(self.model_class, Linker)
|
|
123
|
+
else ModelType.DEDUPER
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
if isinstance(model_settings, dict):
|
|
127
|
+
settings_class = self.model_class.model_fields["settings"].annotation
|
|
128
|
+
self.model_settings = settings_class(**model_settings)
|
|
129
|
+
else:
|
|
130
|
+
self.model_settings = model_settings
|
|
131
|
+
|
|
132
|
+
if (self.model_type == ModelType.LINKER) != (right is not None):
|
|
133
|
+
raise ValueError(
|
|
134
|
+
"A linker requires a right input; a deduper must not have one."
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
upstream: tuple[Step, ...] = (left,) if right is None else (left, right)
|
|
138
|
+
super().__init__(upstream=upstream)
|
|
139
|
+
|
|
140
|
+
# -- Step contract ----------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
def __str__(self) -> str:
|
|
143
|
+
"""A model is drawn with the class implementing it."""
|
|
144
|
+
return f"{self.kind}({self.model_class.__name__})"
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def spec(self) -> ModelSpec:
|
|
148
|
+
"""The serialisable spec for this model."""
|
|
149
|
+
return ModelSpec(
|
|
150
|
+
model_type=self.model_type,
|
|
151
|
+
model_class=self.model_class.__name__,
|
|
152
|
+
model_settings=self.model_settings.model_dump(mode="json"),
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def _execute(self, adapter: Adapter, fp: Fingerprint) -> None:
|
|
156
|
+
left = self.left._read_cache(adapter)
|
|
157
|
+
right = self.right._read_cache(adapter) if self.right else None
|
|
158
|
+
|
|
159
|
+
if self.model_type == ModelType.LINKER:
|
|
160
|
+
self.model_instance.prepare(left, right)
|
|
161
|
+
scores = self.model_instance.link(left=left, right=right)
|
|
162
|
+
else:
|
|
163
|
+
self.model_instance.prepare(left)
|
|
164
|
+
scores = self.model_instance.dedupe(data=left)
|
|
165
|
+
|
|
166
|
+
adapter.store_model(fp, normalise_model_scores(scores))
|
|
167
|
+
|
|
168
|
+
# -- data -------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
def edges(self) -> pl.DataFrame:
|
|
171
|
+
"""Return this model's scored edges. Collects the plan first if needed."""
|
|
172
|
+
if not self.is_collected:
|
|
173
|
+
self.collect()
|
|
174
|
+
adapter, fp = self._collected()
|
|
175
|
+
return adapter.read_model(fp)
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def inputs(self) -> tuple[RecordStep, ...]:
|
|
179
|
+
"""The record steps this model reads."""
|
|
180
|
+
return (self.left,) if self.right is None else (self.left, self.right)
|
|
181
|
+
|
|
182
|
+
# -- verbs ------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
def resolve(
|
|
185
|
+
self,
|
|
186
|
+
*other_models: "Model",
|
|
187
|
+
resolver_class: "type[ResolverMethod] | str" = "Components",
|
|
188
|
+
resolver_settings: "ResolverSettings | dict[str, Any] | None" = None,
|
|
189
|
+
) -> "Resolver":
|
|
190
|
+
"""Resolve this model (and any others) into clusters."""
|
|
191
|
+
from matchlab.resolvers import Resolver # noqa: PLC0415 - avoids a cycle
|
|
192
|
+
|
|
193
|
+
return Resolver(
|
|
194
|
+
self,
|
|
195
|
+
*other_models,
|
|
196
|
+
resolver_class=resolver_class,
|
|
197
|
+
resolver_settings=resolver_settings,
|
|
198
|
+
)
|
matchlab/progress.py
ADDED
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
"""Progress reporting for a collection.
|
|
2
|
+
|
|
3
|
+
`collect()` walks a plan and runs the steps whose artifact is not already stored. This
|
|
4
|
+
module shows that walk while it runs.
|
|
5
|
+
|
|
6
|
+
**The tree is what makes a position mean anything.** Steps have no names. They are
|
|
7
|
+
identified by position (`matchlab.lineage.number`), so a line reading
|
|
8
|
+
`[step 7] Ran in 0.041s` means nothing on its own. You need the tree to know which node
|
|
9
|
+
`7` is. Everything here follows from that. The plan has to be put somewhere findable,
|
|
10
|
+
and the per-step detail has to quote the position.
|
|
11
|
+
|
|
12
|
+
**A collection draws the plan in exactly one place.** Never two.
|
|
13
|
+
|
|
14
|
+
* **A live tree**, at a terminal or in a notebook. One frame, redrawn in place, so a
|
|
15
|
+
fifty-step plan doesn't scroll the shell away. The row that lights up is the row
|
|
16
|
+
numbered `[7]`, so the frame carries both the key and the progress at once. It stays
|
|
17
|
+
on screen throughout, and is left there in full when the run ends, so a `[step 7]` in
|
|
18
|
+
the log beside it always has something to resolve against.
|
|
19
|
+
* **A logged tree**, when nothing is drawn, in CI, a job runner, a redirected stream.
|
|
20
|
+
One multi-line record, the way a traceback is, so it can't be interleaved apart.
|
|
21
|
+
|
|
22
|
+
A drawn tree already answers what a logged one would, so a collection never prints
|
|
23
|
+
both. Printing the plan twice into the same screen would just be noise.
|
|
24
|
+
|
|
25
|
+
The per-step records are logged either way, each prefixed with its position. They land
|
|
26
|
+
on the `matchlab` logger and stay silent unless you have configured logging.
|
|
27
|
+
|
|
28
|
+
Which of the two you get is the `interactive` argument to `Step.collect`, and that
|
|
29
|
+
argument is honoured as given. It is named for the assumption behind it (someone is
|
|
30
|
+
watching) rather than the widget it produces. Drawing means someone is watching, so the
|
|
31
|
+
plan can be a thing on screen that the session throws away. `interactive=False` is what
|
|
32
|
+
puts the tree in the log instead, for a run whose output outlives the session.
|
|
33
|
+
|
|
34
|
+
**Live output and logging share a terminal automatically.** A handler holding a raw
|
|
35
|
+
stream would write straight past Rich and land in the middle of the frame being
|
|
36
|
+
redrawn. That describes the handler `logging.basicConfig()` builds, the one nearly
|
|
37
|
+
every caller has. A report therefore borrows those handlers for as long as it owns the
|
|
38
|
+
terminal, points them at the console, and puts them back afterwards. See
|
|
39
|
+
`matchlab.core.logging.through_console`. Records interleave above the frame instead of
|
|
40
|
+
through it, with no extra setup required. `ConsoleHandler` does the same thing up
|
|
41
|
+
front, for a handler you are building yourself.
|
|
42
|
+
|
|
43
|
+
**A plan taller than the terminal is windowed, not cropped.** Rich would otherwise crop
|
|
44
|
+
a live region to its first rows (`rich.live_render`), which hides the very step you are
|
|
45
|
+
waiting on. The frame instead shows the rows around whichever step is running,
|
|
46
|
+
captioned with how much sits either side. The window follows the run down the tree,
|
|
47
|
+
and the last frame, with nothing left to redraw in place, shows the tree in full.
|
|
48
|
+
|
|
49
|
+
The terminal decides only how much of the frame you see. It has no say in whether a
|
|
50
|
+
tree is drawn, or in what the per-step records say.
|
|
51
|
+
|
|
52
|
+
`Step.collect` decides what runs, and reports what happened. This module only shows it.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
import logging
|
|
56
|
+
import time
|
|
57
|
+
from collections.abc import Iterable, Sequence
|
|
58
|
+
from contextlib import ExitStack
|
|
59
|
+
from contextvars import Token
|
|
60
|
+
from types import TracebackType
|
|
61
|
+
from typing import TYPE_CHECKING
|
|
62
|
+
|
|
63
|
+
from rich.console import Group, RenderableType
|
|
64
|
+
from rich.live import Live
|
|
65
|
+
from rich.text import Text
|
|
66
|
+
|
|
67
|
+
from matchlab.adapters import Adapter, StoreStats
|
|
68
|
+
from matchlab.core import logging as mlog
|
|
69
|
+
from matchlab.lineage import StepState, StepStatus, draw, render
|
|
70
|
+
|
|
71
|
+
if TYPE_CHECKING:
|
|
72
|
+
# `matchlab.steps` imports this module
|
|
73
|
+
from matchlab.steps import Step
|
|
74
|
+
|
|
75
|
+
# The slice of a plan a frame shows, and a note of what it leaves out. `None` when
|
|
76
|
+
# the whole tree fits and nothing is left out.
|
|
77
|
+
_Window = tuple[list[str], Text | None]
|
|
78
|
+
|
|
79
|
+
# Whether a collection is already reporting. Rich allows one live display per console,
|
|
80
|
+
# and a collection can start another collection (`Model.results()` collects if needed).
|
|
81
|
+
# The inner one goes undrawn rather than raising. It still logs — including a tree of
|
|
82
|
+
# its own, immediately before the records that quote it, so its positions are read
|
|
83
|
+
# against its own walk and not against whatever is on screen.
|
|
84
|
+
_REPORTING = False
|
|
85
|
+
|
|
86
|
+
# How each outcome reads in a log line, and at what level. `Cached` is structural
|
|
87
|
+
# rather than work done, so it sits at `DEBUG` and the summary carries its count for
|
|
88
|
+
# an `INFO` reader.
|
|
89
|
+
_RECORDS: dict[StepStatus, tuple[int, str]] = {
|
|
90
|
+
StepStatus.DONE: (logging.INFO, "Ran in {elapsed:.3f}s"),
|
|
91
|
+
StepStatus.CACHED: (logging.DEBUG, "Cached"),
|
|
92
|
+
StepStatus.FAILED: (logging.ERROR, "Failed after {elapsed:.3f}s"),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
# How each status reads in the legend, which has no room to spell it out.
|
|
96
|
+
_LABELS: dict[StepStatus, str] = {
|
|
97
|
+
StepStatus.PENDING: "waiting",
|
|
98
|
+
StepStatus.RUNNING: "running",
|
|
99
|
+
StepStatus.DONE: "ran",
|
|
100
|
+
StepStatus.CACHED: "cached",
|
|
101
|
+
StepStatus.FAILED: "failed",
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Progress:
|
|
106
|
+
"""Receives a collection's events and reports them.
|
|
107
|
+
|
|
108
|
+
One object owns everything a running collection does besides the work itself.
|
|
109
|
+
That includes what is drawn, what is logged, which position a record is
|
|
110
|
+
attributed to, and, while a frame is up, where the application's own handlers
|
|
111
|
+
write. They are grouped together because they must agree. A frame and a log
|
|
112
|
+
disagreeing about which step is running would be worse than either on its own.
|
|
113
|
+
|
|
114
|
+
Used as a context manager by `Step.collect`. Constructing one costs nothing and
|
|
115
|
+
reports nothing. `begin` and `end` drive it.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
def __init__(
|
|
119
|
+
self,
|
|
120
|
+
root: "Step",
|
|
121
|
+
steps: Sequence["Step"],
|
|
122
|
+
*,
|
|
123
|
+
live: bool = False,
|
|
124
|
+
nested: bool = False,
|
|
125
|
+
adapter: Adapter | None = None,
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Prepare a report for collecting `root`, whose plan is `steps`.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
root: The step being collected, the root the tree is drawn from.
|
|
131
|
+
steps: Its plan in walk order, which is what gives each step its position.
|
|
132
|
+
live: Draw the tree as a live frame. Independent of what gets logged.
|
|
133
|
+
nested: Whether an outer collection is already reporting. A nested report
|
|
134
|
+
does not claim the console. It logs as any other does.
|
|
135
|
+
adapter: Where the collection stores what it computes, so the summary can
|
|
136
|
+
say how much room it is taking and how much this run added. Optional,
|
|
137
|
+
since without one the summary simply drops that clause.
|
|
138
|
+
"""
|
|
139
|
+
self.root = root
|
|
140
|
+
self._adapter = adapter
|
|
141
|
+
# The store's size before anything ran, so the summary can report what this
|
|
142
|
+
# collection cost rather than only what the store now weighs. Taken in
|
|
143
|
+
# `__enter__`, since a report that is built and never entered has no run to
|
|
144
|
+
# attribute growth to.
|
|
145
|
+
self._store_before: StoreStats | None = None
|
|
146
|
+
self.steps = tuple(steps)
|
|
147
|
+
# Position by step identity, the same numbering `lineage.number` gives, taken
|
|
148
|
+
# from the walk `collect` already did rather than by walking again.
|
|
149
|
+
self.positions = {
|
|
150
|
+
id(step): position for position, step in enumerate(self.steps)
|
|
151
|
+
}
|
|
152
|
+
self.state: dict[int, StepState] = {
|
|
153
|
+
id(step): StepState(status=StepStatus.PENDING) for step in self.steps
|
|
154
|
+
}
|
|
155
|
+
self._nested = nested
|
|
156
|
+
self._running: Step | None = None
|
|
157
|
+
self._running_since: float | None = None
|
|
158
|
+
self._started: float | None = None
|
|
159
|
+
# Holds the ambient prefix while a step runs, so `begin` can restore whatever
|
|
160
|
+
# was in force on `end`. That matters when this report is nested, since the
|
|
161
|
+
# outer collection's own step prefix was set first.
|
|
162
|
+
self._prefix_token: Token[str | None] | None = None
|
|
163
|
+
# The row a windowed frame centres on. Kept after a step ends rather than
|
|
164
|
+
# cleared, so the view holds still between steps instead of springing back.
|
|
165
|
+
self._focus: int | None = None
|
|
166
|
+
# Whether the collection is over, and the frame can stop being a frame.
|
|
167
|
+
self._finished = False
|
|
168
|
+
# Everything this report has to undo, the frame and the log handlers it
|
|
169
|
+
# borrowed to keep off it. `None` before `__enter__` and after `__exit__`.
|
|
170
|
+
self._stack: ExitStack | None = None
|
|
171
|
+
# Which row each step is drawn on. A plan's shape doesn't change while it
|
|
172
|
+
# collects, only the markers along it, so this is fixed for the run.
|
|
173
|
+
# Computed only when there is a frame to window.
|
|
174
|
+
self._rows: dict[int, int] = render(root)[1] if live else {}
|
|
175
|
+
# Set last, because Rich draws a first frame while constructing the display.
|
|
176
|
+
self._live = (
|
|
177
|
+
Live(
|
|
178
|
+
get_renderable=self._renderable,
|
|
179
|
+
console=mlog.console,
|
|
180
|
+
refresh_per_second=8,
|
|
181
|
+
)
|
|
182
|
+
if live
|
|
183
|
+
else None
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# -- events ---------------------------------------------------------------------
|
|
187
|
+
|
|
188
|
+
def begin(self, step: "Step") -> None:
|
|
189
|
+
"""Mark `step` as the one now running, and publish its position.
|
|
190
|
+
|
|
191
|
+
Publishing the position is what attributes the records the step's own code
|
|
192
|
+
emits, a linker counting matches, a source reading the warehouse. That code
|
|
193
|
+
cannot name its own position, because a position belongs to the walk, not the
|
|
194
|
+
step. The walk holds it here, and passes it on.
|
|
195
|
+
"""
|
|
196
|
+
self._release()
|
|
197
|
+
self._running = step
|
|
198
|
+
self._running_since = time.perf_counter()
|
|
199
|
+
self.state[id(step)] = StepState(status=StepStatus.RUNNING)
|
|
200
|
+
self._prefix_token = mlog.prefix.set(f"step {self.positions[id(step)]}")
|
|
201
|
+
self._focus = self._rows.get(id(step))
|
|
202
|
+
self._refresh()
|
|
203
|
+
|
|
204
|
+
def end(self, step: "Step", status: StepStatus) -> None:
|
|
205
|
+
"""Record how `step` finished, how long it took, and log a line for it."""
|
|
206
|
+
elapsed = (
|
|
207
|
+
time.perf_counter() - self._running_since
|
|
208
|
+
if self._running_since is not None
|
|
209
|
+
else 0.0
|
|
210
|
+
)
|
|
211
|
+
self._running = None
|
|
212
|
+
self._running_since = None
|
|
213
|
+
self.state[id(step)] = StepState(status=status, elapsed=elapsed)
|
|
214
|
+
self._refresh()
|
|
215
|
+
|
|
216
|
+
# Logged before the prefix is released, so this record is tagged like the ones
|
|
217
|
+
# the step itself emitted.
|
|
218
|
+
level, template = _RECORDS[status]
|
|
219
|
+
mlog.logger.log(level, template.format(elapsed=elapsed))
|
|
220
|
+
self._release()
|
|
221
|
+
|
|
222
|
+
def _release(self) -> None:
|
|
223
|
+
"""Restore the prefix that was in force before the running step claimed it."""
|
|
224
|
+
if self._prefix_token is not None:
|
|
225
|
+
mlog.prefix.reset(self._prefix_token)
|
|
226
|
+
self._prefix_token = None
|
|
227
|
+
|
|
228
|
+
# -- lifecycle ------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
def __enter__(self) -> "Progress":
|
|
231
|
+
"""Start reporting, putting the plan on screen or in the log."""
|
|
232
|
+
global _REPORTING # one report per console
|
|
233
|
+
self._started = time.perf_counter()
|
|
234
|
+
if self._adapter is not None:
|
|
235
|
+
self._store_before = self._adapter.stats()
|
|
236
|
+
stack = ExitStack()
|
|
237
|
+
try:
|
|
238
|
+
if self._live is not None:
|
|
239
|
+
# The redirect first, so the streams it reads are the real ones rather
|
|
240
|
+
# than the proxies Rich is about to install over them. Unwinding is
|
|
241
|
+
# last-in-first-out, which is also the order teardown wants. Stop the
|
|
242
|
+
# frame, *then* hand the handlers back.
|
|
243
|
+
stack.enter_context(mlog.through_console())
|
|
244
|
+
self._live.start(refresh=True)
|
|
245
|
+
stack.callback(self._stop)
|
|
246
|
+
else:
|
|
247
|
+
# The tree once, up front. The records that follow quote positions,
|
|
248
|
+
# and this is what says which node each position is. A drawn tree is
|
|
249
|
+
# already that key. It stays on screen throughout, and is left in full
|
|
250
|
+
# when the run ends, so logging a second copy beside it would only
|
|
251
|
+
# print the plan twice.
|
|
252
|
+
mlog.logger.info(f"Collecting {len(self.steps)} steps:\n{self.tree()}")
|
|
253
|
+
except BaseException:
|
|
254
|
+
# Rich raises out of `start` if its first frame fails. Without this the
|
|
255
|
+
# handlers it borrowed would stay borrowed for the life of the process,
|
|
256
|
+
# since a failed `__enter__` means `__exit__` never runs.
|
|
257
|
+
stack.close()
|
|
258
|
+
raise
|
|
259
|
+
self._stack = stack
|
|
260
|
+
if not self._nested:
|
|
261
|
+
_REPORTING = True
|
|
262
|
+
return self
|
|
263
|
+
|
|
264
|
+
def __exit__(
|
|
265
|
+
self,
|
|
266
|
+
exc_type: type[BaseException] | None,
|
|
267
|
+
exc: BaseException | None,
|
|
268
|
+
traceback: TracebackType | None,
|
|
269
|
+
) -> None:
|
|
270
|
+
"""Stop reporting, leaving the finished tree on screen."""
|
|
271
|
+
global _REPORTING # one report per console
|
|
272
|
+
# Whatever happened, this report is no longer the one tagging records. An
|
|
273
|
+
# exception out of `collect` leaves the running step's `end` uncalled.
|
|
274
|
+
self._release()
|
|
275
|
+
try:
|
|
276
|
+
if self._stack is not None:
|
|
277
|
+
self._stack.close()
|
|
278
|
+
self._stack = None
|
|
279
|
+
finally:
|
|
280
|
+
# Even if teardown failed. Otherwise the flag would stay set, and every
|
|
281
|
+
# later collection would treat itself as nested and skip drawing.
|
|
282
|
+
if not self._nested:
|
|
283
|
+
_REPORTING = False
|
|
284
|
+
if exc_type is None:
|
|
285
|
+
mlog.logger.info(self.summary())
|
|
286
|
+
|
|
287
|
+
def _stop(self) -> None:
|
|
288
|
+
"""Take the frame down, leaving the finished plan on screen."""
|
|
289
|
+
if self._live is None:
|
|
290
|
+
return
|
|
291
|
+
# Set before the stop, so the frame Rich draws on its way out is the whole tree
|
|
292
|
+
# rather than the window the run was watching through.
|
|
293
|
+
self._finished = True
|
|
294
|
+
self._live.stop()
|
|
295
|
+
# Drop the display, breaking the reference cycle it forms with this object
|
|
296
|
+
# (`Live` holds `self._renderable`, a bound method, and this holds `Live`).
|
|
297
|
+
# This object holds every step, so the cycle would keep a whole plan, and
|
|
298
|
+
# through it the adapter, alive until the cyclic collector happened to run.
|
|
299
|
+
# Releasing here lets the plan die with `collect`'s frame, by refcount.
|
|
300
|
+
self._live = None
|
|
301
|
+
|
|
302
|
+
def tree(self) -> str:
|
|
303
|
+
"""The plan, as a tree, in whatever state the collection has reached."""
|
|
304
|
+
return draw(self.root, self.state)
|
|
305
|
+
|
|
306
|
+
def summary(self) -> str:
|
|
307
|
+
"""One line describing what the collection did, and what it cost to keep.
|
|
308
|
+
|
|
309
|
+
Carries the cached count because those per-step records sit at `DEBUG`. This
|
|
310
|
+
is where an `INFO` reader learns what the run skipped.
|
|
311
|
+
|
|
312
|
+
It also carries the store's size, because nothing else does. A store keeps
|
|
313
|
+
everything collected into it and lives in a cache directory nobody opens, so
|
|
314
|
+
the first sign of it is a full disk with nothing to blame. The delta is the
|
|
315
|
+
part that assigns blame. It says what *this* run added, which is what connects
|
|
316
|
+
a size to the edit that caused it.
|
|
317
|
+
"""
|
|
318
|
+
counts = self._counts(self.state)
|
|
319
|
+
elapsed = time.perf_counter() - self._started if self._started else 0.0
|
|
320
|
+
line = (
|
|
321
|
+
f"Collected {len(self.steps)} steps ("
|
|
322
|
+
f"{counts[StepStatus.DONE]} ran, "
|
|
323
|
+
f"{counts[StepStatus.CACHED]} cached) in {elapsed:.3f}s"
|
|
324
|
+
)
|
|
325
|
+
if self._adapter is None:
|
|
326
|
+
return line
|
|
327
|
+
|
|
328
|
+
# The store says this part itself, because what is worth saying about it
|
|
329
|
+
# depends on the backend. `DuckDBStoreStats` distinguishes bytes in memory
|
|
330
|
+
# from bytes on a disk, and another store will have its own qualification.
|
|
331
|
+
return f"{line}. {self._adapter.stats().describe(since=self._store_before)}"
|
|
332
|
+
|
|
333
|
+
# -- rendering ------------------------------------------------------------------
|
|
334
|
+
|
|
335
|
+
def _refresh(self) -> None:
|
|
336
|
+
if self._live is not None:
|
|
337
|
+
self._live.refresh()
|
|
338
|
+
|
|
339
|
+
def _renderable(self) -> Group:
|
|
340
|
+
"""Draw the current frame. Called by Rich on every refresh."""
|
|
341
|
+
state = self._snapshot()
|
|
342
|
+
# Rows come from `self._rows`, fixed at construction. State moves the markers
|
|
343
|
+
# along a plan, never the shape of it.
|
|
344
|
+
lines = render(self.root, state, markup=True)[0]
|
|
345
|
+
|
|
346
|
+
# The last frame is the whole tree. It is the thing left on screen, it is what
|
|
347
|
+
# the summary refers to, and letting it scroll costs nothing once there is no
|
|
348
|
+
# frame left to redraw in place. Rich does the same at `Live.stop`.
|
|
349
|
+
if self._finished:
|
|
350
|
+
return Group(Text.from_markup("\n".join(lines)), _legend(state.values()))
|
|
351
|
+
|
|
352
|
+
window, elided = self._window(lines)
|
|
353
|
+
parts: list[RenderableType] = [Text.from_markup("\n".join(window))]
|
|
354
|
+
if elided is not None:
|
|
355
|
+
parts.insert(0, elided)
|
|
356
|
+
parts.append(_legend(state.values()))
|
|
357
|
+
return Group(*parts)
|
|
358
|
+
|
|
359
|
+
def _window(self, lines: Sequence[str]) -> _Window:
|
|
360
|
+
"""The rows to show, and a note of what that leaves out.
|
|
361
|
+
|
|
362
|
+
A plan taller than the terminal cannot be shown whole in a frame redrawn in
|
|
363
|
+
place, but it doesn't have to be. What a reader needs from a running plan is
|
|
364
|
+
the step that is running and the structure around it, so the frame follows it,
|
|
365
|
+
keeping it centred until the ends of the tree clamp the window.
|
|
366
|
+
|
|
367
|
+
Returns:
|
|
368
|
+
`(rows, note)`, the slice to draw, and a line saying how much is hidden
|
|
369
|
+
either side, or `None` when the whole tree fits and nothing is.
|
|
370
|
+
"""
|
|
371
|
+
height = mlog.console.size.height
|
|
372
|
+
if len(lines) <= height - 1: # the legend is the only other claim on the space
|
|
373
|
+
return list(lines), None
|
|
374
|
+
|
|
375
|
+
budget = max(1, height - 2) # the legend, and the note this returns
|
|
376
|
+
focus = self._focus if self._focus is not None else 0
|
|
377
|
+
# Centred on the running row, then clamped, so the window stops at the ends of
|
|
378
|
+
# the tree rather than running past them into blank space.
|
|
379
|
+
start = max(0, min(focus - budget // 2, len(lines) - budget))
|
|
380
|
+
above, below = start, len(lines) - (start + budget)
|
|
381
|
+
|
|
382
|
+
hidden = [
|
|
383
|
+
f"{count} more {where}"
|
|
384
|
+
for where, count in (("above", above), ("below", below))
|
|
385
|
+
if count
|
|
386
|
+
]
|
|
387
|
+
note = Text(f"⋮ {' · '.join(hidden)}", style="dim", no_wrap=True)
|
|
388
|
+
return list(lines[start : start + budget]), note
|
|
389
|
+
|
|
390
|
+
def _snapshot(self) -> dict[int, StepState]:
|
|
391
|
+
"""The state to draw, with the running step's elapsed time brought up to date.
|
|
392
|
+
|
|
393
|
+
Recomputed per frame rather than stored, so the running step's timer ticks
|
|
394
|
+
between events instead of freezing on whatever it read when the step started.
|
|
395
|
+
"""
|
|
396
|
+
if self._running is None or self._running_since is None:
|
|
397
|
+
return self.state
|
|
398
|
+
running = time.perf_counter() - self._running_since
|
|
399
|
+
return self.state | {
|
|
400
|
+
id(self._running): StepState(status=StepStatus.RUNNING, elapsed=running),
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
@staticmethod
|
|
404
|
+
def _counts(state: dict[int, StepState]) -> dict[StepStatus, int]:
|
|
405
|
+
counts = {status: 0 for status in StepStatus}
|
|
406
|
+
for step_state in state.values():
|
|
407
|
+
counts[step_state.status] += 1
|
|
408
|
+
return counts
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _legend(states: Iterable[StepState]) -> Text:
|
|
412
|
+
"""A key to the markers, covering only the statuses actually on screen.
|
|
413
|
+
|
|
414
|
+
The glyphs aren't self-explanatory, and `cached` versus `ran` is the thing a reader
|
|
415
|
+
most wants to know. Listing only what's present keeps it to one line and lets it
|
|
416
|
+
shrink as the run settles.
|
|
417
|
+
"""
|
|
418
|
+
present = {state.status for state in states}
|
|
419
|
+
legend = Text(no_wrap=True, overflow="ellipsis")
|
|
420
|
+
for status in StepStatus:
|
|
421
|
+
if status not in present:
|
|
422
|
+
continue
|
|
423
|
+
if legend:
|
|
424
|
+
legend.append(" ", style="dim")
|
|
425
|
+
legend.append(status.marker, style=status.style)
|
|
426
|
+
legend.append(f" {_LABELS[status]}", style="dim")
|
|
427
|
+
return legend
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def report(
|
|
431
|
+
root: "Step",
|
|
432
|
+
steps: Sequence["Step"],
|
|
433
|
+
interactive: bool | None,
|
|
434
|
+
adapter: Adapter | None = None,
|
|
435
|
+
) -> Progress:
|
|
436
|
+
"""Build the reporter for collecting `root`.
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
root: The step being collected.
|
|
440
|
+
steps: Its plan, in walk order.
|
|
441
|
+
interactive: Whether someone is watching, so the plan should be drawn rather
|
|
442
|
+
than just logged. `None`, the default, takes a terminal or a notebook as
|
|
443
|
+
a yes, and anything else as a no. How tall the plan is doesn't enter into
|
|
444
|
+
it. One too tall to show is windowed, not cropped.
|
|
445
|
+
adapter: Where the collection stores what it computes, so the summary can
|
|
446
|
+
report the store's size and what this run added to it.
|
|
447
|
+
"""
|
|
448
|
+
if interactive is None:
|
|
449
|
+
live = mlog.console.is_terminal or mlog.console.is_jupyter
|
|
450
|
+
else:
|
|
451
|
+
live = interactive
|
|
452
|
+
# Rich allows one live display per console, so an inner collection goes undrawn
|
|
453
|
+
# whatever it asked for. It still logs, tree and all.
|
|
454
|
+
live = live and not _REPORTING
|
|
455
|
+
return Progress(root, steps, live=live, nested=_REPORTING, adapter=adapter)
|