matchlab 0.1.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. matchlab/__init__.py +55 -0
  2. matchlab/adapters/__init__.py +24 -0
  3. matchlab/adapters/base.py +446 -0
  4. matchlab/adapters/duckdb.py +659 -0
  5. matchlab/cli.py +177 -0
  6. matchlab/core/__init__.py +1 -0
  7. matchlab/core/dataframes.py +254 -0
  8. matchlab/core/dsu.py +62 -0
  9. matchlab/core/exceptions.py +40 -0
  10. matchlab/core/hash.py +141 -0
  11. matchlab/core/kinds.py +12 -0
  12. matchlab/core/logging.py +190 -0
  13. matchlab/core/resolver_output.py +173 -0
  14. matchlab/core/schemas.py +82 -0
  15. matchlab/core/sql.py +12 -0
  16. matchlab/document.py +329 -0
  17. matchlab/eval/__init__.py +11 -0
  18. matchlab/eval/judgements.py +41 -0
  19. matchlab/eval/metrics.py +244 -0
  20. matchlab/eval/samples.py +415 -0
  21. matchlab/eval/tui/__init__.py +66 -0
  22. matchlab/eval/tui/app.py +418 -0
  23. matchlab/eval/tui/modals.py +84 -0
  24. matchlab/eval/tui/styles.tcss +184 -0
  25. matchlab/eval/tui/widgets/__init__.py +1 -0
  26. matchlab/eval/tui/widgets/assignment.py +72 -0
  27. matchlab/eval/tui/widgets/styling.py +75 -0
  28. matchlab/eval/tui/widgets/table.py +285 -0
  29. matchlab/lineage.py +239 -0
  30. matchlab/locations.py +347 -0
  31. matchlab/models/__init__.py +10 -0
  32. matchlab/models/comparison.py +67 -0
  33. matchlab/models/dedupers/__init__.py +5 -0
  34. matchlab/models/dedupers/base.py +38 -0
  35. matchlab/models/dedupers/naive.py +77 -0
  36. matchlab/models/linkers/__init__.py +9 -0
  37. matchlab/models/linkers/base.py +41 -0
  38. matchlab/models/linkers/deterministic.py +229 -0
  39. matchlab/models/linkers/splinklinker.py +262 -0
  40. matchlab/models/linkers/weighteddeterministic.py +143 -0
  41. matchlab/models/models.py +198 -0
  42. matchlab/progress.py +455 -0
  43. matchlab/recordstep.py +181 -0
  44. matchlab/resolvers/__init__.py +10 -0
  45. matchlab/resolvers/base.py +63 -0
  46. matchlab/resolvers/components.py +66 -0
  47. matchlab/resolvers/resolvers.py +409 -0
  48. matchlab/sources.py +291 -0
  49. matchlab/specs.py +129 -0
  50. matchlab/steps.py +284 -0
  51. matchlab/testkit/__init__.py +104 -0
  52. matchlab/testkit/_generate.py +240 -0
  53. matchlab/testkit/compare.py +178 -0
  54. matchlab/testkit/entities.py +290 -0
  55. matchlab/testkit/features.py +146 -0
  56. matchlab/testkit/linked.py +527 -0
  57. matchlab/testkit/matchers.py +233 -0
  58. matchlab/testkit/models.py +165 -0
  59. matchlab/testkit/sources.py +327 -0
  60. matchlab/transformers/__init__.py +24 -0
  61. matchlab/transformers/base.py +41 -0
  62. matchlab/transformers/clean.py +48 -0
  63. matchlab/transformers/explode.py +67 -0
  64. matchlab/transformers/group.py +56 -0
  65. matchlab/transformers/select.py +31 -0
  66. matchlab/transformers/transform.py +101 -0
  67. matchlab-0.1.0.dev1.dist-info/METADATA +88 -0
  68. matchlab-0.1.0.dev1.dist-info/RECORD +72 -0
  69. matchlab-0.1.0.dev1.dist-info/WHEEL +5 -0
  70. matchlab-0.1.0.dev1.dist-info/entry_points.txt +2 -0
  71. matchlab-0.1.0.dev1.dist-info/licenses/LICENSE +21 -0
  72. matchlab-0.1.0.dev1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,198 @@
1
+ """Model — a deduper or linker producing scored candidate edges."""
2
+
3
+ import inspect
4
+ from typing import TYPE_CHECKING, Any, ClassVar
5
+
6
+ import polars as pl
7
+
8
+ from matchlab.adapters import Adapter, Fingerprint
9
+ from matchlab.core.kinds import StepKind
10
+ from matchlab.core.logging import logger
11
+ from matchlab.core.schemas import SCHEMA_MODEL_EDGES
12
+ from matchlab.models import dedupers, linkers
13
+ from matchlab.models.dedupers.base import Deduper, DeduperSettings
14
+ from matchlab.models.linkers.base import Linker, LinkerSettings
15
+ from matchlab.recordstep import RecordStep
16
+ from matchlab.specs import ModelSpec, ModelType
17
+ from matchlab.steps import Step
18
+
19
+ if TYPE_CHECKING:
20
+ # `matchlab.resolvers` imports this module
21
+ from matchlab.resolvers import Resolver
22
+ from matchlab.resolvers.base import ResolverMethod, ResolverSettings
23
+
24
+ _MODEL_CLASSES: dict[str, type[Linker] | type[Deduper]] = {
25
+ **dict(inspect.getmembers(dedupers, inspect.isclass)),
26
+ **dict(inspect.getmembers(linkers, inspect.isclass)),
27
+ }
28
+
29
+
30
+ def add_model_class(model_class: type[Linker] | type[Deduper]) -> None:
31
+ """Register a custom deduper or linker so it can be named in a plan."""
32
+ if not issubclass(model_class, Linker | Deduper):
33
+ raise ValueError("The argument is not a subclass of Deduper or Linker.")
34
+ _MODEL_CLASSES[model_class.__name__] = model_class
35
+
36
+
37
+ def normalise_model_scores(scores: pl.DataFrame) -> pl.DataFrame:
38
+ """Validate a methodology's raw output and cast it to `SCHEMA_MODEL_EDGES`.
39
+
40
+ Raises `ValueError` if `scores` isn't a `DataFrame` with exactly `left_id`,
41
+ `right_id`, and a numeric `score` in `[0.0, 1.0]`. An unordered pair can appear
42
+ twice, once as `(a, b)` and once as `(b, a)`. When that happens, this keeps only
43
+ the highest-scoring row and logs a warning, since a resolver expects one edge per
44
+ pair.
45
+ """
46
+ if not isinstance(scores, pl.DataFrame):
47
+ raise ValueError(f"Expected a polars DataFrame, got {type(scores)}.")
48
+
49
+ expected_fields = set(SCHEMA_MODEL_EDGES.names)
50
+ if set(scores.columns) != expected_fields:
51
+ raise ValueError(f"Expected {expected_fields}.\nFound {set(scores.columns)}.")
52
+
53
+ if scores.height == 0:
54
+ scores = pl.DataFrame(schema=pl.Schema(SCHEMA_MODEL_EDGES))
55
+
56
+ if not scores["score"].dtype.is_numeric():
57
+ raise ValueError(
58
+ "Score column must contain numeric values in the range [0.0, 1.0]."
59
+ )
60
+
61
+ normalised_scores = scores.with_columns(pl.col("score").cast(pl.Float32))
62
+ invalid_scores = normalised_scores.filter(
63
+ pl.col("score").is_null()
64
+ | pl.col("score").is_nan()
65
+ | (pl.col("score") < 0.0)
66
+ | (pl.col("score") > 1.0)
67
+ )
68
+ if invalid_scores.height:
69
+ min_score = normalised_scores["score"].min()
70
+ max_score = normalised_scores["score"].max()
71
+ raise ValueError(f"Score range misconfigured: [{min_score}, {max_score}]")
72
+
73
+ unique_scores = (
74
+ normalised_scores.with_columns(
75
+ pl.concat_list(
76
+ [pl.col("left_id").cast(pl.Utf8), pl.col("right_id").cast(pl.Utf8)]
77
+ )
78
+ .list.sort()
79
+ .list.join("_")
80
+ .alias("sorted_ids")
81
+ )
82
+ .sort("score", descending=True) # sort so largest score comes first
83
+ .unique(
84
+ subset=["sorted_ids"], keep="first"
85
+ ) # keep first occurrence after sorting
86
+ ).drop("sorted_ids")
87
+ if len(scores) != len(unique_scores):
88
+ logger.warning("Duplicate pairs! Keeping only pairs with highest score.")
89
+
90
+ return unique_scores.cast(pl.Schema(SCHEMA_MODEL_EDGES))
91
+
92
+
93
+ class Model(Step):
94
+ """A step that runs one methodology (a Deduper or a Linker) over record steps."""
95
+
96
+ kind: ClassVar[StepKind] = StepKind.MODEL
97
+
98
+ def __init__(
99
+ self,
100
+ left: RecordStep,
101
+ model_class: type[Deduper] | type[Linker] | str,
102
+ model_settings: DeduperSettings | LinkerSettings | dict,
103
+ right: RecordStep | None = None,
104
+ ) -> None:
105
+ """Define a model.
106
+
107
+ Args:
108
+ left: The record step to deduplicate, or the left side of a link.
109
+ model_class: A `Deduper`/`Linker` subclass, or its registered name.
110
+ model_settings: The settings object for that class, or a dict.
111
+ right: The right side of a link. Omit for a deduper.
112
+ """
113
+ self.left = left
114
+ self.right = right
115
+
116
+ self.model_class = (
117
+ _MODEL_CLASSES[model_class] if isinstance(model_class, str) else model_class
118
+ )
119
+ self.model_instance = self.model_class(settings=model_settings)
120
+ self.model_type = (
121
+ ModelType.LINKER
122
+ if issubclass(self.model_class, Linker)
123
+ else ModelType.DEDUPER
124
+ )
125
+
126
+ if isinstance(model_settings, dict):
127
+ settings_class = self.model_class.model_fields["settings"].annotation
128
+ self.model_settings = settings_class(**model_settings)
129
+ else:
130
+ self.model_settings = model_settings
131
+
132
+ if (self.model_type == ModelType.LINKER) != (right is not None):
133
+ raise ValueError(
134
+ "A linker requires a right input; a deduper must not have one."
135
+ )
136
+
137
+ upstream: tuple[Step, ...] = (left,) if right is None else (left, right)
138
+ super().__init__(upstream=upstream)
139
+
140
+ # -- Step contract ----------------------------------------------------------------
141
+
142
+ def __str__(self) -> str:
143
+ """A model is drawn with the class implementing it."""
144
+ return f"{self.kind}({self.model_class.__name__})"
145
+
146
+ @property
147
+ def spec(self) -> ModelSpec:
148
+ """The serialisable spec for this model."""
149
+ return ModelSpec(
150
+ model_type=self.model_type,
151
+ model_class=self.model_class.__name__,
152
+ model_settings=self.model_settings.model_dump(mode="json"),
153
+ )
154
+
155
+ def _execute(self, adapter: Adapter, fp: Fingerprint) -> None:
156
+ left = self.left._read_cache(adapter)
157
+ right = self.right._read_cache(adapter) if self.right else None
158
+
159
+ if self.model_type == ModelType.LINKER:
160
+ self.model_instance.prepare(left, right)
161
+ scores = self.model_instance.link(left=left, right=right)
162
+ else:
163
+ self.model_instance.prepare(left)
164
+ scores = self.model_instance.dedupe(data=left)
165
+
166
+ adapter.store_model(fp, normalise_model_scores(scores))
167
+
168
+ # -- data -------------------------------------------------------------------------
169
+
170
+ def edges(self) -> pl.DataFrame:
171
+ """Return this model's scored edges. Collects the plan first if needed."""
172
+ if not self.is_collected:
173
+ self.collect()
174
+ adapter, fp = self._collected()
175
+ return adapter.read_model(fp)
176
+
177
+ @property
178
+ def inputs(self) -> tuple[RecordStep, ...]:
179
+ """The record steps this model reads."""
180
+ return (self.left,) if self.right is None else (self.left, self.right)
181
+
182
+ # -- verbs ------------------------------------------------------------------------
183
+
184
+ def resolve(
185
+ self,
186
+ *other_models: "Model",
187
+ resolver_class: "type[ResolverMethod] | str" = "Components",
188
+ resolver_settings: "ResolverSettings | dict[str, Any] | None" = None,
189
+ ) -> "Resolver":
190
+ """Resolve this model (and any others) into clusters."""
191
+ from matchlab.resolvers import Resolver # noqa: PLC0415 - avoids a cycle
192
+
193
+ return Resolver(
194
+ self,
195
+ *other_models,
196
+ resolver_class=resolver_class,
197
+ resolver_settings=resolver_settings,
198
+ )
matchlab/progress.py ADDED
@@ -0,0 +1,455 @@
1
+ """Progress reporting for a collection.
2
+
3
+ `collect()` walks a plan and runs the steps whose artifact is not already stored. This
4
+ module shows that walk while it runs.
5
+
6
+ **The tree is what makes a position mean anything.** Steps have no names. They are
7
+ identified by position (`matchlab.lineage.number`), so a line reading
8
+ `[step 7] Ran in 0.041s` means nothing on its own. You need the tree to know which node
9
+ `7` is. Everything here follows from that. The plan has to be put somewhere findable,
10
+ and the per-step detail has to quote the position.
11
+
12
+ **A collection draws the plan in exactly one place.** Never two.
13
+
14
+ * **A live tree**, at a terminal or in a notebook. One frame, redrawn in place, so a
15
+ fifty-step plan doesn't scroll the shell away. The row that lights up is the row
16
+ numbered `[7]`, so the frame carries both the key and the progress at once. It stays
17
+ on screen throughout, and is left there in full when the run ends, so a `[step 7]` in
18
+ the log beside it always has something to resolve against.
19
+ * **A logged tree**, when nothing is drawn, in CI, a job runner, a redirected stream.
20
+ One multi-line record, the way a traceback is, so it can't be interleaved apart.
21
+
22
+ A drawn tree already answers what a logged one would, so a collection never prints
23
+ both. Printing the plan twice into the same screen would just be noise.
24
+
25
+ The per-step records are logged either way, each prefixed with its position. They land
26
+ on the `matchlab` logger and stay silent unless you have configured logging.
27
+
28
+ Which of the two you get is the `interactive` argument to `Step.collect`, and that
29
+ argument is honoured as given. It is named for the assumption behind it (someone is
30
+ watching) rather than the widget it produces. Drawing means someone is watching, so the
31
+ plan can be a thing on screen that the session throws away. `interactive=False` is what
32
+ puts the tree in the log instead, for a run whose output outlives the session.
33
+
34
+ **Live output and logging share a terminal automatically.** A handler holding a raw
35
+ stream would write straight past Rich and land in the middle of the frame being
36
+ redrawn. That describes the handler `logging.basicConfig()` builds, the one nearly
37
+ every caller has. A report therefore borrows those handlers for as long as it owns the
38
+ terminal, points them at the console, and puts them back afterwards. See
39
+ `matchlab.core.logging.through_console`. Records interleave above the frame instead of
40
+ through it, with no extra setup required. `ConsoleHandler` does the same thing up
41
+ front, for a handler you are building yourself.
42
+
43
+ **A plan taller than the terminal is windowed, not cropped.** Rich would otherwise crop
44
+ a live region to its first rows (`rich.live_render`), which hides the very step you are
45
+ waiting on. The frame instead shows the rows around whichever step is running,
46
+ captioned with how much sits either side. The window follows the run down the tree,
47
+ and the last frame, with nothing left to redraw in place, shows the tree in full.
48
+
49
+ The terminal decides only how much of the frame you see. It has no say in whether a
50
+ tree is drawn, or in what the per-step records say.
51
+
52
+ `Step.collect` decides what runs, and reports what happened. This module only shows it.
53
+ """
54
+
55
+ import logging
56
+ import time
57
+ from collections.abc import Iterable, Sequence
58
+ from contextlib import ExitStack
59
+ from contextvars import Token
60
+ from types import TracebackType
61
+ from typing import TYPE_CHECKING
62
+
63
+ from rich.console import Group, RenderableType
64
+ from rich.live import Live
65
+ from rich.text import Text
66
+
67
+ from matchlab.adapters import Adapter, StoreStats
68
+ from matchlab.core import logging as mlog
69
+ from matchlab.lineage import StepState, StepStatus, draw, render
70
+
71
+ if TYPE_CHECKING:
72
+ # `matchlab.steps` imports this module
73
+ from matchlab.steps import Step
74
+
75
+ # The slice of a plan a frame shows, and a note of what it leaves out. `None` when
76
+ # the whole tree fits and nothing is left out.
77
+ _Window = tuple[list[str], Text | None]
78
+
79
+ # Whether a collection is already reporting. Rich allows one live display per console,
80
+ # and a collection can start another collection (`Model.results()` collects if needed).
81
+ # The inner one goes undrawn rather than raising. It still logs — including a tree of
82
+ # its own, immediately before the records that quote it, so its positions are read
83
+ # against its own walk and not against whatever is on screen.
84
+ _REPORTING = False
85
+
86
+ # How each outcome reads in a log line, and at what level. `Cached` is structural
87
+ # rather than work done, so it sits at `DEBUG` and the summary carries its count for
88
+ # an `INFO` reader.
89
+ _RECORDS: dict[StepStatus, tuple[int, str]] = {
90
+ StepStatus.DONE: (logging.INFO, "Ran in {elapsed:.3f}s"),
91
+ StepStatus.CACHED: (logging.DEBUG, "Cached"),
92
+ StepStatus.FAILED: (logging.ERROR, "Failed after {elapsed:.3f}s"),
93
+ }
94
+
95
+ # How each status reads in the legend, which has no room to spell it out.
96
+ _LABELS: dict[StepStatus, str] = {
97
+ StepStatus.PENDING: "waiting",
98
+ StepStatus.RUNNING: "running",
99
+ StepStatus.DONE: "ran",
100
+ StepStatus.CACHED: "cached",
101
+ StepStatus.FAILED: "failed",
102
+ }
103
+
104
+
105
+ class Progress:
106
+ """Receives a collection's events and reports them.
107
+
108
+ One object owns everything a running collection does besides the work itself.
109
+ That includes what is drawn, what is logged, which position a record is
110
+ attributed to, and, while a frame is up, where the application's own handlers
111
+ write. They are grouped together because they must agree. A frame and a log
112
+ disagreeing about which step is running would be worse than either on its own.
113
+
114
+ Used as a context manager by `Step.collect`. Constructing one costs nothing and
115
+ reports nothing. `begin` and `end` drive it.
116
+ """
117
+
118
+ def __init__(
119
+ self,
120
+ root: "Step",
121
+ steps: Sequence["Step"],
122
+ *,
123
+ live: bool = False,
124
+ nested: bool = False,
125
+ adapter: Adapter | None = None,
126
+ ) -> None:
127
+ """Prepare a report for collecting `root`, whose plan is `steps`.
128
+
129
+ Args:
130
+ root: The step being collected, the root the tree is drawn from.
131
+ steps: Its plan in walk order, which is what gives each step its position.
132
+ live: Draw the tree as a live frame. Independent of what gets logged.
133
+ nested: Whether an outer collection is already reporting. A nested report
134
+ does not claim the console. It logs as any other does.
135
+ adapter: Where the collection stores what it computes, so the summary can
136
+ say how much room it is taking and how much this run added. Optional,
137
+ since without one the summary simply drops that clause.
138
+ """
139
+ self.root = root
140
+ self._adapter = adapter
141
+ # The store's size before anything ran, so the summary can report what this
142
+ # collection cost rather than only what the store now weighs. Taken in
143
+ # `__enter__`, since a report that is built and never entered has no run to
144
+ # attribute growth to.
145
+ self._store_before: StoreStats | None = None
146
+ self.steps = tuple(steps)
147
+ # Position by step identity, the same numbering `lineage.number` gives, taken
148
+ # from the walk `collect` already did rather than by walking again.
149
+ self.positions = {
150
+ id(step): position for position, step in enumerate(self.steps)
151
+ }
152
+ self.state: dict[int, StepState] = {
153
+ id(step): StepState(status=StepStatus.PENDING) for step in self.steps
154
+ }
155
+ self._nested = nested
156
+ self._running: Step | None = None
157
+ self._running_since: float | None = None
158
+ self._started: float | None = None
159
+ # Holds the ambient prefix while a step runs, so `begin` can restore whatever
160
+ # was in force on `end`. That matters when this report is nested, since the
161
+ # outer collection's own step prefix was set first.
162
+ self._prefix_token: Token[str | None] | None = None
163
+ # The row a windowed frame centres on. Kept after a step ends rather than
164
+ # cleared, so the view holds still between steps instead of springing back.
165
+ self._focus: int | None = None
166
+ # Whether the collection is over, and the frame can stop being a frame.
167
+ self._finished = False
168
+ # Everything this report has to undo, the frame and the log handlers it
169
+ # borrowed to keep off it. `None` before `__enter__` and after `__exit__`.
170
+ self._stack: ExitStack | None = None
171
+ # Which row each step is drawn on. A plan's shape doesn't change while it
172
+ # collects, only the markers along it, so this is fixed for the run.
173
+ # Computed only when there is a frame to window.
174
+ self._rows: dict[int, int] = render(root)[1] if live else {}
175
+ # Set last, because Rich draws a first frame while constructing the display.
176
+ self._live = (
177
+ Live(
178
+ get_renderable=self._renderable,
179
+ console=mlog.console,
180
+ refresh_per_second=8,
181
+ )
182
+ if live
183
+ else None
184
+ )
185
+
186
+ # -- events ---------------------------------------------------------------------
187
+
188
+ def begin(self, step: "Step") -> None:
189
+ """Mark `step` as the one now running, and publish its position.
190
+
191
+ Publishing the position is what attributes the records the step's own code
192
+ emits, a linker counting matches, a source reading the warehouse. That code
193
+ cannot name its own position, because a position belongs to the walk, not the
194
+ step. The walk holds it here, and passes it on.
195
+ """
196
+ self._release()
197
+ self._running = step
198
+ self._running_since = time.perf_counter()
199
+ self.state[id(step)] = StepState(status=StepStatus.RUNNING)
200
+ self._prefix_token = mlog.prefix.set(f"step {self.positions[id(step)]}")
201
+ self._focus = self._rows.get(id(step))
202
+ self._refresh()
203
+
204
+ def end(self, step: "Step", status: StepStatus) -> None:
205
+ """Record how `step` finished, how long it took, and log a line for it."""
206
+ elapsed = (
207
+ time.perf_counter() - self._running_since
208
+ if self._running_since is not None
209
+ else 0.0
210
+ )
211
+ self._running = None
212
+ self._running_since = None
213
+ self.state[id(step)] = StepState(status=status, elapsed=elapsed)
214
+ self._refresh()
215
+
216
+ # Logged before the prefix is released, so this record is tagged like the ones
217
+ # the step itself emitted.
218
+ level, template = _RECORDS[status]
219
+ mlog.logger.log(level, template.format(elapsed=elapsed))
220
+ self._release()
221
+
222
+ def _release(self) -> None:
223
+ """Restore the prefix that was in force before the running step claimed it."""
224
+ if self._prefix_token is not None:
225
+ mlog.prefix.reset(self._prefix_token)
226
+ self._prefix_token = None
227
+
228
+ # -- lifecycle ------------------------------------------------------------------
229
+
230
+ def __enter__(self) -> "Progress":
231
+ """Start reporting, putting the plan on screen or in the log."""
232
+ global _REPORTING # one report per console
233
+ self._started = time.perf_counter()
234
+ if self._adapter is not None:
235
+ self._store_before = self._adapter.stats()
236
+ stack = ExitStack()
237
+ try:
238
+ if self._live is not None:
239
+ # The redirect first, so the streams it reads are the real ones rather
240
+ # than the proxies Rich is about to install over them. Unwinding is
241
+ # last-in-first-out, which is also the order teardown wants. Stop the
242
+ # frame, *then* hand the handlers back.
243
+ stack.enter_context(mlog.through_console())
244
+ self._live.start(refresh=True)
245
+ stack.callback(self._stop)
246
+ else:
247
+ # The tree once, up front. The records that follow quote positions,
248
+ # and this is what says which node each position is. A drawn tree is
249
+ # already that key. It stays on screen throughout, and is left in full
250
+ # when the run ends, so logging a second copy beside it would only
251
+ # print the plan twice.
252
+ mlog.logger.info(f"Collecting {len(self.steps)} steps:\n{self.tree()}")
253
+ except BaseException:
254
+ # Rich raises out of `start` if its first frame fails. Without this the
255
+ # handlers it borrowed would stay borrowed for the life of the process,
256
+ # since a failed `__enter__` means `__exit__` never runs.
257
+ stack.close()
258
+ raise
259
+ self._stack = stack
260
+ if not self._nested:
261
+ _REPORTING = True
262
+ return self
263
+
264
+ def __exit__(
265
+ self,
266
+ exc_type: type[BaseException] | None,
267
+ exc: BaseException | None,
268
+ traceback: TracebackType | None,
269
+ ) -> None:
270
+ """Stop reporting, leaving the finished tree on screen."""
271
+ global _REPORTING # one report per console
272
+ # Whatever happened, this report is no longer the one tagging records. An
273
+ # exception out of `collect` leaves the running step's `end` uncalled.
274
+ self._release()
275
+ try:
276
+ if self._stack is not None:
277
+ self._stack.close()
278
+ self._stack = None
279
+ finally:
280
+ # Even if teardown failed. Otherwise the flag would stay set, and every
281
+ # later collection would treat itself as nested and skip drawing.
282
+ if not self._nested:
283
+ _REPORTING = False
284
+ if exc_type is None:
285
+ mlog.logger.info(self.summary())
286
+
287
+ def _stop(self) -> None:
288
+ """Take the frame down, leaving the finished plan on screen."""
289
+ if self._live is None:
290
+ return
291
+ # Set before the stop, so the frame Rich draws on its way out is the whole tree
292
+ # rather than the window the run was watching through.
293
+ self._finished = True
294
+ self._live.stop()
295
+ # Drop the display, breaking the reference cycle it forms with this object
296
+ # (`Live` holds `self._renderable`, a bound method, and this holds `Live`).
297
+ # This object holds every step, so the cycle would keep a whole plan, and
298
+ # through it the adapter, alive until the cyclic collector happened to run.
299
+ # Releasing here lets the plan die with `collect`'s frame, by refcount.
300
+ self._live = None
301
+
302
+ def tree(self) -> str:
303
+ """The plan, as a tree, in whatever state the collection has reached."""
304
+ return draw(self.root, self.state)
305
+
306
+ def summary(self) -> str:
307
+ """One line describing what the collection did, and what it cost to keep.
308
+
309
+ Carries the cached count because those per-step records sit at `DEBUG`. This
310
+ is where an `INFO` reader learns what the run skipped.
311
+
312
+ It also carries the store's size, because nothing else does. A store keeps
313
+ everything collected into it and lives in a cache directory nobody opens, so
314
+ the first sign of it is a full disk with nothing to blame. The delta is the
315
+ part that assigns blame. It says what *this* run added, which is what connects
316
+ a size to the edit that caused it.
317
+ """
318
+ counts = self._counts(self.state)
319
+ elapsed = time.perf_counter() - self._started if self._started else 0.0
320
+ line = (
321
+ f"Collected {len(self.steps)} steps ("
322
+ f"{counts[StepStatus.DONE]} ran, "
323
+ f"{counts[StepStatus.CACHED]} cached) in {elapsed:.3f}s"
324
+ )
325
+ if self._adapter is None:
326
+ return line
327
+
328
+ # The store says this part itself, because what is worth saying about it
329
+ # depends on the backend. `DuckDBStoreStats` distinguishes bytes in memory
330
+ # from bytes on a disk, and another store will have its own qualification.
331
+ return f"{line}. {self._adapter.stats().describe(since=self._store_before)}"
332
+
333
+ # -- rendering ------------------------------------------------------------------
334
+
335
+ def _refresh(self) -> None:
336
+ if self._live is not None:
337
+ self._live.refresh()
338
+
339
+ def _renderable(self) -> Group:
340
+ """Draw the current frame. Called by Rich on every refresh."""
341
+ state = self._snapshot()
342
+ # Rows come from `self._rows`, fixed at construction. State moves the markers
343
+ # along a plan, never the shape of it.
344
+ lines = render(self.root, state, markup=True)[0]
345
+
346
+ # The last frame is the whole tree. It is the thing left on screen, it is what
347
+ # the summary refers to, and letting it scroll costs nothing once there is no
348
+ # frame left to redraw in place. Rich does the same at `Live.stop`.
349
+ if self._finished:
350
+ return Group(Text.from_markup("\n".join(lines)), _legend(state.values()))
351
+
352
+ window, elided = self._window(lines)
353
+ parts: list[RenderableType] = [Text.from_markup("\n".join(window))]
354
+ if elided is not None:
355
+ parts.insert(0, elided)
356
+ parts.append(_legend(state.values()))
357
+ return Group(*parts)
358
+
359
+ def _window(self, lines: Sequence[str]) -> _Window:
360
+ """The rows to show, and a note of what that leaves out.
361
+
362
+ A plan taller than the terminal cannot be shown whole in a frame redrawn in
363
+ place, but it doesn't have to be. What a reader needs from a running plan is
364
+ the step that is running and the structure around it, so the frame follows it,
365
+ keeping it centred until the ends of the tree clamp the window.
366
+
367
+ Returns:
368
+ `(rows, note)`, the slice to draw, and a line saying how much is hidden
369
+ either side, or `None` when the whole tree fits and nothing is.
370
+ """
371
+ height = mlog.console.size.height
372
+ if len(lines) <= height - 1: # the legend is the only other claim on the space
373
+ return list(lines), None
374
+
375
+ budget = max(1, height - 2) # the legend, and the note this returns
376
+ focus = self._focus if self._focus is not None else 0
377
+ # Centred on the running row, then clamped, so the window stops at the ends of
378
+ # the tree rather than running past them into blank space.
379
+ start = max(0, min(focus - budget // 2, len(lines) - budget))
380
+ above, below = start, len(lines) - (start + budget)
381
+
382
+ hidden = [
383
+ f"{count} more {where}"
384
+ for where, count in (("above", above), ("below", below))
385
+ if count
386
+ ]
387
+ note = Text(f"⋮ {' · '.join(hidden)}", style="dim", no_wrap=True)
388
+ return list(lines[start : start + budget]), note
389
+
390
+ def _snapshot(self) -> dict[int, StepState]:
391
+ """The state to draw, with the running step's elapsed time brought up to date.
392
+
393
+ Recomputed per frame rather than stored, so the running step's timer ticks
394
+ between events instead of freezing on whatever it read when the step started.
395
+ """
396
+ if self._running is None or self._running_since is None:
397
+ return self.state
398
+ running = time.perf_counter() - self._running_since
399
+ return self.state | {
400
+ id(self._running): StepState(status=StepStatus.RUNNING, elapsed=running),
401
+ }
402
+
403
+ @staticmethod
404
+ def _counts(state: dict[int, StepState]) -> dict[StepStatus, int]:
405
+ counts = {status: 0 for status in StepStatus}
406
+ for step_state in state.values():
407
+ counts[step_state.status] += 1
408
+ return counts
409
+
410
+
411
+ def _legend(states: Iterable[StepState]) -> Text:
412
+ """A key to the markers, covering only the statuses actually on screen.
413
+
414
+ The glyphs aren't self-explanatory, and `cached` versus `ran` is the thing a reader
415
+ most wants to know. Listing only what's present keeps it to one line and lets it
416
+ shrink as the run settles.
417
+ """
418
+ present = {state.status for state in states}
419
+ legend = Text(no_wrap=True, overflow="ellipsis")
420
+ for status in StepStatus:
421
+ if status not in present:
422
+ continue
423
+ if legend:
424
+ legend.append(" ", style="dim")
425
+ legend.append(status.marker, style=status.style)
426
+ legend.append(f" {_LABELS[status]}", style="dim")
427
+ return legend
428
+
429
+
430
+ def report(
431
+ root: "Step",
432
+ steps: Sequence["Step"],
433
+ interactive: bool | None,
434
+ adapter: Adapter | None = None,
435
+ ) -> Progress:
436
+ """Build the reporter for collecting `root`.
437
+
438
+ Args:
439
+ root: The step being collected.
440
+ steps: Its plan, in walk order.
441
+ interactive: Whether someone is watching, so the plan should be drawn rather
442
+ than just logged. `None`, the default, takes a terminal or a notebook as
443
+ a yes, and anything else as a no. How tall the plan is doesn't enter into
444
+ it. One too tall to show is windowed, not cropped.
445
+ adapter: Where the collection stores what it computes, so the summary can
446
+ report the store's size and what this run added to it.
447
+ """
448
+ if interactive is None:
449
+ live = mlog.console.is_terminal or mlog.console.is_jupyter
450
+ else:
451
+ live = interactive
452
+ # Rich allows one live display per console, so an inner collection goes undrawn
453
+ # whatever it asked for. It still logs, tree and all.
454
+ live = live and not _REPORTING
455
+ return Progress(root, steps, live=live, nested=_REPORTING, adapter=adapter)