quantology-modelmaker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. modelmaker/__init__.py +2 -0
  2. modelmaker/api.py +1073 -0
  3. modelmaker/blocks/__init__.py +0 -0
  4. modelmaker/blocks/base.py +78 -0
  5. modelmaker/blocks/library.py +327 -0
  6. modelmaker/blocks/modelling.py +312 -0
  7. modelmaker/blocks/stat_tests.py +133 -0
  8. modelmaker/cache.py +44 -0
  9. modelmaker/compiler.py +478 -0
  10. modelmaker/graph.py +171 -0
  11. modelmaker/llm/__init__.py +28 -0
  12. modelmaker/llm/anthropic_provider.py +79 -0
  13. modelmaker/llm/base.py +85 -0
  14. modelmaker/llm/claude_cli_provider.py +90 -0
  15. modelmaker/llm/gemini_provider.py +138 -0
  16. modelmaker/llm/lmstudio_provider.py +140 -0
  17. modelmaker/llm/openai_provider.py +138 -0
  18. modelmaker/llm/prompts.py +189 -0
  19. modelmaker/llm/settings.py +47 -0
  20. modelmaker/llm/stub_provider.py +29 -0
  21. modelmaker/metadata_transforms.py +109 -0
  22. modelmaker/packet.py +111 -0
  23. modelmaker/project.py +129 -0
  24. modelmaker/run_worker.py +143 -0
  25. modelmaker/runner.py +1313 -0
  26. modelmaker/session.py +400 -0
  27. modelmaker/static/assets/index-B9cHjTMW.css +1 -0
  28. modelmaker/static/assets/index-EvjTc-1k.js +34 -0
  29. modelmaker/static/brand-quantology/favicon.svg +4 -0
  30. modelmaker/static/brand-quantology/icon-mark-light.svg +13 -0
  31. modelmaker/static/icons.svg +24 -0
  32. modelmaker/static/index.html +20 -0
  33. modelmaker/tui/__init__.py +0 -0
  34. modelmaker/tui/app.py +785 -0
  35. modelmaker/tui/canvas.py +388 -0
  36. modelmaker/tui/charts.py +84 -0
  37. modelmaker/tui/client.py +270 -0
  38. modelmaker/tui/inspector.py +285 -0
  39. modelmaker/tui/paramspecs.py +128 -0
  40. modelmaker/tui/preview.py +71 -0
  41. modelmaker/tui/screens.py +182 -0
  42. modelmaker/util.py +44 -0
  43. quantology_modelmaker-0.1.0.dist-info/METADATA +247 -0
  44. quantology_modelmaker-0.1.0.dist-info/RECORD +49 -0
  45. quantology_modelmaker-0.1.0.dist-info/WHEEL +5 -0
  46. quantology_modelmaker-0.1.0.dist-info/entry_points.txt +3 -0
  47. quantology_modelmaker-0.1.0.dist-info/licenses/LICENSE +204 -0
  48. quantology_modelmaker-0.1.0.dist-info/licenses/NOTICE +12 -0
  49. quantology_modelmaker-0.1.0.dist-info/top_level.txt +1 -0
modelmaker/compiler.py ADDED
@@ -0,0 +1,478 @@
1
+ from __future__ import annotations
2
+
3
+ import inspect
4
+ import textwrap
5
+ from datetime import datetime, timezone
6
+
7
+ from .blocks.base import BLOCK_REGISTRY, BlockSpec
8
+ from .graph import Graph
9
+ from .packet import ColumnRole, DataFramePacket, resolve_role_column
10
+ from .runner import FusionGroup, _is_fusable_block
11
+ from .util import ROLE_PARAM_NAMES, accepts_param, find_role_param
12
+
13
+
14
+ class CompileError(Exception):
15
+ pass
16
+
17
+
18
+ def _sanitize(name: str) -> str:
19
+ out = "".join(c if c.isalnum() or c == "_" else "_" for c in name)
20
+ return out if out and not out[0].isdigit() else f"_{out}"
21
+
22
+
23
+ def _alloc_output_vars(
24
+ bid: str,
25
+ block,
26
+ out_ports: list[str],
27
+ var_names: dict[tuple[str, str], str],
28
+ used_var_names: set[str],
29
+ ) -> list[str]:
30
+ """Pick the compiled script's variable name(s) for `bid`'s output
31
+ port(s) -- a user-chosen port name (see BlockInstance.port_names) when
32
+ one's set and not already taken, else the block-name/id/port scheme,
33
+ which is always unique by construction. Shared between the ordinary
34
+ per-block call emission and a fused group's exit(s) (see
35
+ _fused_group_call_lines) so a name picked for a streaming exit reads
36
+ exactly like any other block's output would."""
37
+ varlist = []
38
+ for p in out_ports:
39
+ custom = block.port_names.get(p)
40
+ var = _sanitize(custom) if custom else None
41
+ if var is None or var in used_var_names:
42
+ var = f"{_sanitize(block.name)}_{bid}" if len(out_ports) == 1 else f"{_sanitize(block.name)}_{bid}_{p}"
43
+ used_var_names.add(var)
44
+ var_names[(bid, p)] = var
45
+ varlist.append(var)
46
+ return varlist
47
+
48
+
49
+ def _build_compile_fusion_groups(graph: Graph, order: list[str], must_exit: set[str]) -> list[FusionGroup]:
50
+ """Compiler-side counterpart of Runner._build_fusion_groups (see
51
+ runner.py for the full design): groups `order` (the whole reachable
52
+ compile set -- a compiled script unconditionally (re)executes
53
+ everything it emits, so there's no "not yet current" subset the way a
54
+ live run has) into fusion groups exactly as FusionGroup describes,
55
+ using dataframe wires restricted to `order` itself -- a block outside
56
+ what this compile emits doesn't exist as far as the script is
57
+ concerned, unlike the live engine where an already-cached consumer
58
+ outside the run still matters for a future run.
59
+
60
+ `must_exit` (an explicitly requested `output_blocks` entry) forces a
61
+ real, individually-named exit even when the block's only consumer is
62
+ also being compiled -- the whole point of naming a block in
63
+ `output_blocks` is to get its own variable in the script."""
64
+ candidate_set = set(order)
65
+ fusable: dict[str, BlockSpec] = {}
66
+ for bid in order:
67
+ spec = _is_fusable_block(graph.blocks[bid])
68
+ if spec is not None:
69
+ fusable[bid] = spec
70
+
71
+ parent: dict[str, str] = {bid: bid for bid in fusable}
72
+
73
+ def find(x: str) -> str:
74
+ while parent[x] != x:
75
+ parent[x] = parent[parent[x]]
76
+ x = parent[x]
77
+ return x
78
+
79
+ def union(a: str, b: str) -> None:
80
+ ra, rb = find(a), find(b)
81
+ if ra != rb:
82
+ parent[ra] = rb
83
+
84
+ port_types_cache: dict[str, dict[str, str]] = {}
85
+
86
+ def port_types(bid: str) -> dict[str, str]:
87
+ if bid not in port_types_cache:
88
+ port_types_cache[bid] = {p.name: p.type for p in graph.blocks[bid].inputs}
89
+ return port_types_cache[bid]
90
+
91
+ for bid in fusable:
92
+ for port, wire in graph.input_wires(bid).items():
93
+ if port_types(bid).get(port) != "dataframe":
94
+ continue
95
+ if wire.from_block in fusable:
96
+ union(bid, wire.from_block)
97
+
98
+ groups_by_root: dict[str, list[str]] = {}
99
+ for bid in order: # preserves topo order
100
+ key = find(bid) if bid in fusable else bid
101
+ groups_by_root.setdefault(key, []).append(bid)
102
+
103
+ groups: list[FusionGroup] = []
104
+ for members in groups_by_root.values():
105
+ member_set = set(members)
106
+ if len(members) == 1 and members[0] not in fusable:
107
+ groups.append(FusionGroup(members=list(members), exits=set(member_set)))
108
+ continue
109
+ exits: set[str] = set()
110
+ for bid in members:
111
+ out_ports = {p.name for p in graph.blocks[bid].outputs if p.type == "dataframe"}
112
+ df_consumers = [
113
+ w
114
+ for w in graph.wires.values()
115
+ if w.from_block == bid
116
+ and w.from_port in out_ports
117
+ and w.to_block in candidate_set
118
+ and port_types(w.to_block).get(w.to_port) == "dataframe"
119
+ ]
120
+ if not df_consumers or any(w.to_block not in member_set for w in df_consumers) or bid in must_exit:
121
+ exits.add(bid)
122
+ groups.append(FusionGroup(members=list(members), exits=exits))
123
+ return groups
124
+
125
+
126
+ def _fused_group_call_lines(
127
+ group: FusionGroup,
128
+ graph: Graph,
129
+ fn_names: dict[str, str],
130
+ var_names: dict[tuple[str, str], str],
131
+ used_var_names: set[str],
132
+ ) -> list[str]:
133
+ """Emit one fusion group (see _build_compile_fusion_groups) as a
134
+ single polars lazy plan: every member becomes a `_lz_<id> = fn(...)`
135
+ assignment (chained inputs referencing another member's own `_lz_`
136
+ variable directly; anything else -- an already-compiled eager
137
+ DataFrame from outside the group -- wrapped in `.lazy()` so it joins
138
+ the same unbroken plan), and the group's exit(s) are collected
139
+ together in one `pl.collect_all(..., engine="streaming")` call, which
140
+ applies common-subplan elimination across them so a shared upstream
141
+ stretch (fan-out) is computed once even when it feeds two exits."""
142
+ member_set = set(group.members)
143
+ lines: list[str] = [f"# --- Fused (streaming): {', '.join(group.members)} ---"]
144
+ lazy_var = {bid: f"_lz_{_sanitize(bid)}" for bid in group.members}
145
+
146
+ for bid in group.members:
147
+ block = graph.blocks[bid]
148
+ kwarg_pairs: list[tuple[str, str]] = []
149
+ for port, wire in sorted(graph.input_wires(bid).items()):
150
+ if wire.from_block in member_set:
151
+ kwarg_pairs.append((port, lazy_var[wire.from_block]))
152
+ else:
153
+ kwarg_pairs.append((port, f"{var_names[(wire.from_block, wire.from_port)]}.lazy()"))
154
+ for pname, pval in block.params.items():
155
+ kwarg_pairs.append((pname, repr(pval)))
156
+ call = f"{fn_names[bid]}({', '.join(f'{k}={v}' for k, v in kwarg_pairs)})"
157
+ lines.append(f"{lazy_var[bid]} = {call}")
158
+
159
+ exits = [bid for bid in group.members if bid in group.exits]
160
+ exit_vars = [_alloc_output_vars(bid, graph.blocks[bid], ["out"], var_names, used_var_names)[0] for bid in exits]
161
+ collect_call = f"pl.collect_all([{', '.join(lazy_var[bid] for bid in exits)}], engine='streaming')"
162
+ if len(exit_vars) == 1:
163
+ lines.append(f"({exit_vars[0]},) = {collect_call}")
164
+ else:
165
+ lines.append(f"{', '.join(exit_vars)} = {collect_call}")
166
+ return lines
167
+
168
+
169
+ def _resolve_role_value(graph: Graph, runner, block_id: str, role: ColumnRole) -> str | None:
170
+ """Same dynamic default as Runner.run_block (see packet.resolve_role_column),
171
+ computed here from the already-cached, already-green upstream packets
172
+ `runner` holds (compiling requires every reachable block to be green
173
+ first -- see the not_ready check above) rather than re-running anything.
174
+ Retagging a column's role changes the block's cache key (see
175
+ Runner.compute_key), so a block whose target/predicted column just moved
176
+ is grey again and (in strict mode) blocks compilation until it's re-run
177
+ -- by the time this runs, the cached packet always reflects the current
178
+ tag."""
179
+ if runner is None:
180
+ return None
181
+ schema_metas = []
182
+ for wire in graph.input_wires(block_id).values():
183
+ st = runner.state.get(wire.from_block)
184
+ if st is None or st.last_successful_key is None:
185
+ continue
186
+ entry = runner.cache.get(st.last_successful_key)
187
+ if entry is None:
188
+ continue
189
+ packet = entry.outputs.get(wire.from_port)
190
+ if isinstance(packet, DataFramePacket):
191
+ schema_metas.append(packet.schema_meta)
192
+ return resolve_role_column(schema_metas, role)
193
+
194
+
195
+ def compile_graph(
196
+ graph: Graph,
197
+ runner=None,
198
+ output_blocks: list[str] | None = None,
199
+ strict: bool = True,
200
+ stream: bool = False,
201
+ ) -> str:
202
+ """Compile the graph to a single, self-contained Python script (plan
203
+ section 7, default/clean mode — DataFramePacket/ColumnMeta scaffolding
204
+ is absent by construction, since block bodies are plain-dataframe
205
+ functions to begin with). `--with-metadata` verbose mode is not yet
206
+ implemented.
207
+
208
+ `stream=True` mirrors Runner.run_all_streaming for a compiled script:
209
+ whatever connected stretch of fusable blocks (filter/select/
210
+ groupby_agg/join/read_csv -- see BlockSpec.lazy_fn, and
211
+ _build_compile_fusion_groups for the exact same grouping rule the
212
+ live engine uses) is emitted as one polars lazy plan, collected once
213
+ via `pl.collect_all(..., engine="streaming")`, instead of one eager
214
+ call per block -- so a compiled script's source, not just the live
215
+ engine, can process a source larger than memory. Default False keeps
216
+ every existing compiled script byte-for-byte identical to before this
217
+ existed."""
218
+ targets = output_blocks or list(graph.blocks)
219
+ reachable = graph.ancestors(targets)
220
+
221
+ if runner is not None:
222
+ not_ready = sorted(b for b in reachable if runner.status(b) in ("grey", "red"))
223
+ if not_ready and strict:
224
+ raise CompileError(f"blocks not ready to compile (grey/red): {not_ready}")
225
+
226
+ order = [b for b in graph.topo_order() if b in reachable]
227
+
228
+ # Streaming mode groups the compile set up front so both passes below
229
+ # (function defs, then call sites) can tell a fused member from an
230
+ # ordinary one -- must_exit only applies when output_blocks was given
231
+ # explicitly (an unscoped compile has no extra "name this variable"
232
+ # requirement beyond normal graph consumption, which the grouping's
233
+ # own no-consumers-outside-the-group rule already covers).
234
+ fusion_groups = _build_compile_fusion_groups(graph, order, set(output_blocks or [])) if stream else []
235
+ fused_group_of: dict[str, FusionGroup] = {}
236
+ for group in fusion_groups:
237
+ if len(group.members) > 1:
238
+ for bid in group.members:
239
+ fused_group_of[bid] = group
240
+
241
+ fn_names: dict[str, str] = {}
242
+ wants_output_dir: dict[str, bool] = {}
243
+ wants_block_id: dict[str, bool] = {}
244
+ wants_role_params: dict[str, dict[ColumnRole, str]] = {}
245
+ def_lines: list[str] = []
246
+ # Two blocks that run the same code (same category, for registry blocks;
247
+ # same category *and* code text, for custom ones) get exactly one
248
+ # function definition, shared across every instance's call site below --
249
+ # e.g. two WoE blocks compile to one `woe_transform` function called
250
+ # twice with each instance's own params, not two near-identical copies.
251
+ # A fused member's def uses its lazy_fn instead of the eager fn (see
252
+ # BlockSpec.lazy_fn) -- for filter/select/groupby_agg/join that's
253
+ # literally the same function object, so the emitted source is
254
+ # unchanged; only read_csv's separate lazy twin actually differs, and
255
+ # naturally gets its own def (different source text -> different key
256
+ # below) alongside any non-fused read_csv elsewhere in the script.
257
+ seen_fns: dict[tuple[str, str], str] = {}
258
+ used_fn_names: set[str] = set()
259
+ for bid in order:
260
+ block = graph.blocks[bid]
261
+ group = fused_group_of.get(bid)
262
+ if group is not None:
263
+ fn = BLOCK_REGISTRY[block.category].lazy_fn
264
+ else:
265
+ fn = block.resolved_fn()
266
+ wants_output_dir[bid] = accepts_param(fn, "output_dir")
267
+ wants_block_id[bid] = accepts_param(fn, "block_id")
268
+ wants_role_params[bid] = {
269
+ role: role_param for role in ROLE_PARAM_NAMES if (role_param := find_role_param(fn, role)) is not None
270
+ }
271
+ src = block.code if block.is_custom else inspect.getsource(fn)
272
+ src = textwrap.dedent(src).strip("\n")
273
+ fn_sig = (block.category, src)
274
+ existing_fn_name = seen_fns.get(fn_sig)
275
+ if existing_fn_name is not None:
276
+ fn_names[bid] = existing_fn_name
277
+ continue
278
+
279
+ # A custom (AI-authored) block's function is named after the block
280
+ # itself -- the same human-chosen name already used for its call-site
281
+ # variable below -- rather than its auto-generated category (e.g.
282
+ # "ai_block_lz3k9f"), so the compiled def reads like the block does
283
+ # in the UI. Registry blocks keep the plain category name (a stable,
284
+ # well-known function name shared by every instance); a different
285
+ # *bodied* block already claiming it only bites two custom blocks
286
+ # that happen to share a category but were drafted with different
287
+ # bodies, where the block id disambiguates them.
288
+ base_name = _sanitize(block.name) if block.is_custom else _sanitize(block.category)
289
+ fn_name = base_name if base_name not in used_fn_names else f"{base_name}_{bid}"
290
+ used_fn_names.add(fn_name)
291
+ seen_fns[fn_sig] = fn_name
292
+ fn_names[bid] = fn_name
293
+ src = src.replace(f"def {fn.__name__}(", f"def {fn_name}(", 1)
294
+ def_lines.append(f"# === Function: {fn_name} | type={block.block_type} | category={block.category} ===")
295
+ def_lines.append(src)
296
+ def_lines.append("")
297
+ def_lines.append("")
298
+
299
+ call_lines: list[str] = []
300
+ var_names: dict[tuple[str, str], str] = {}
301
+ used_var_names: set[str] = set()
302
+ current_lane: str | None = "__unset__"
303
+ uses_group_by = any(graph.blocks[bid].group_by for bid in order)
304
+ emitted_groups: set[int] = set()
305
+ for bid in order:
306
+ block = graph.blocks[bid]
307
+ # Lanes group the pipeline into modeling phases (Data Prep -> Feature
308
+ # Engineering -> ...); a banner marks each phase's calls in the
309
+ # compiled script, purely for readability -- lane membership plays no
310
+ # role in execution order (see graph.topo_order).
311
+ lane_name = graph.lanes[block.lane].name if block.lane in graph.lanes else None
312
+ if lane_name != current_lane:
313
+ current_lane = lane_name
314
+ if lane_name:
315
+ call_lines.append(f"# ===== Lane: {lane_name} =====")
316
+
317
+ group = fused_group_of.get(bid)
318
+ if group is not None:
319
+ if id(group) in emitted_groups:
320
+ continue # already emitted in full when we reached the group's first member
321
+ emitted_groups.add(id(group))
322
+ call_lines.extend(_fused_group_call_lines(group, graph, fn_names, var_names, used_var_names))
323
+ call_lines.append("")
324
+ continue
325
+
326
+ # (kwarg name, source-code expression for its value) pairs -- kept
327
+ # apart instead of pre-joined into "name=value" text so grouped
328
+ # blocks (below) can split dataframe-typed wired inputs (candidates
329
+ # for partitioning) from everything else (params, role defaults,
330
+ # output_dir/block_id -- unchanged across groups).
331
+ dataframe_ports = {p.name for p in block.inputs if p.type == "dataframe"}
332
+ kwarg_pairs: list[tuple[str, str]] = []
333
+ for port, wire in sorted(graph.input_wires(bid).items()):
334
+ kwarg_pairs.append((port, var_names[(wire.from_block, wire.from_port)]))
335
+ for pname, pval in block.params.items():
336
+ kwarg_pairs.append((pname, repr(pval)))
337
+ for role, role_param in wants_role_params[bid].items():
338
+ if role_param not in block.params:
339
+ resolved = _resolve_role_value(graph, runner, bid, role)
340
+ if resolved is not None:
341
+ kwarg_pairs.append((role_param, repr(resolved)))
342
+ if block.block_type == "output" and wants_output_dir[bid]:
343
+ kwarg_pairs.append(("output_dir", "OUTPUT_DIR"))
344
+ if block.block_type == "output" and wants_block_id[bid]:
345
+ kwarg_pairs.append(("block_id", repr(bid)))
346
+
347
+ out_ports = [p.name for p in block.outputs]
348
+ call_lines.append(f'# --- Call: {bid} | name="{block.name}" ---')
349
+
350
+ # A port the user's named (see BlockInstance.port_names, settable
351
+ # by clicking that output's data in the UI) becomes the variable
352
+ # holding it here, so the compiled script reads with the same
353
+ # names the user gave the data -- falling back to the
354
+ # block-name/id/port scheme, which is always unique by
355
+ # construction, for any port left unnamed or whose chosen name
356
+ # collides with another one already used in this script.
357
+ varlist = _alloc_output_vars(bid, block, out_ports, var_names, used_var_names)
358
+
359
+ if block.group_by:
360
+ call_lines.extend(_grouped_call_lines(bid, block, fn_names[bid], dataframe_ports, kwarg_pairs, varlist))
361
+ else:
362
+ call = f"{fn_names[bid]}({', '.join(f'{k}={v}' for k, v in kwarg_pairs)})"
363
+ if not out_ports:
364
+ call_lines.append(call)
365
+ else:
366
+ call_lines.append(f"{', '.join(varlist)} = {call}" if len(varlist) > 1 else f"{varlist[0]} = {call}")
367
+ call_lines.append("")
368
+
369
+ header = [
370
+ "# Generated by Model-Maker. Do not edit function bodies here directly --",
371
+ "# edit the source blocks in the project and recompile.",
372
+ f"# Compiled: {datetime.now(timezone.utc).isoformat()}",
373
+ "",
374
+ "import os",
375
+ "import polars as pl",
376
+ ]
377
+ if uses_group_by:
378
+ header.append("from concurrent.futures import ThreadPoolExecutor")
379
+ header += [
380
+ "",
381
+ 'OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "./output")',
382
+ "os.makedirs(OUTPUT_DIR, exist_ok=True)",
383
+ "",
384
+ "",
385
+ ]
386
+ if uses_group_by:
387
+ header.append(_COMBINE_GROUP_RESULTS_SRC)
388
+ header.append("")
389
+
390
+ return "\n".join(header + def_lines + call_lines) + "\n"
391
+
392
+
393
+ # A grouped block's per-group calls each return whatever the block normally
394
+ # returns (a single value, or a tuple for a multi-output block) -- this
395
+ # reshapes {group_value: that return value} into the single/tuple result
396
+ # the rest of the script expects, exactly mirroring Runner._combine_group_results
397
+ # so a compiled script's grouped output matches the live engine's. Emitted
398
+ # once, only when at least one block in the graph uses group_by.
399
+ _COMBINE_GROUP_RESULTS_SRC = '''def _combine_group_results(group_col, group_values, raw_by_group, out_names):
400
+ if not out_names:
401
+ return None
402
+ per_group = {}
403
+ for gval in group_values:
404
+ raw = raw_by_group[gval]
405
+ per_group[gval] = (raw,) if len(out_names) == 1 else tuple(raw)
406
+ combined = []
407
+ for i in range(len(out_names)):
408
+ values = [per_group[g][i] for g in group_values]
409
+ if all(isinstance(v, pl.DataFrame) for v in values):
410
+ parts = [
411
+ v if group_col in v.columns else v.with_columns(pl.lit(gval).alias(group_col))
412
+ for gval, v in zip(group_values, values)
413
+ ]
414
+ combined.append(pl.concat(parts, how="diagonal_relaxed"))
415
+ elif all(isinstance(v, dict) for v in values):
416
+ combined.append(pl.DataFrame([{group_col: gval, **v} for gval, v in zip(group_values, values)]))
417
+ else:
418
+ combined.append(dict(zip(group_values, values)))
419
+ return combined[0] if len(combined) == 1 else tuple(combined)
420
+ '''
421
+
422
+
423
+ def _grouped_call_lines(
424
+ bid: str,
425
+ block,
426
+ fn_name: str,
427
+ dataframe_ports: set[str],
428
+ kwarg_pairs: list[tuple[str, str]],
429
+ varlist: list[str],
430
+ ) -> list[str]:
431
+ """Emit a group_by block's call as a partition/dispatch/recombine
432
+ sequence instead of a single call -- mirrors Runner._run_grouped, but
433
+ threaded (ThreadPoolExecutor) rather than process-isolated: this is a
434
+ one-shot batch script the user runs themselves, not a long-lived server
435
+ process other work needs protecting from, so the extra isolation the
436
+ live app pays for isn't worth the portability cost here (a compiled
437
+ script using real subprocesses would need every grouped block wrapped
438
+ in `if __name__ == "__main__":` to be safe under Windows' default
439
+ "spawn" start method)."""
440
+ df_pairs = [(k, v) for k, v in kwarg_pairs if k in dataframe_ports]
441
+ other_pairs = [(k, v) for k, v in kwarg_pairs if k not in dataframe_ports]
442
+ group_col = block.group_by
443
+ max_workers = block.max_workers or 4
444
+ suffix = bid
445
+
446
+ lines = [
447
+ f"_df_kwargs_{suffix} = {{{', '.join(f'{k!r}: {v}' for k, v in df_pairs)}}}",
448
+ f"_fixed_kwargs_{suffix} = {{{', '.join(f'{k!r}: {v}' for k, v in other_pairs)}}}",
449
+ f"_groupable_{suffix} = {{k: v for k, v in _df_kwargs_{suffix}.items() if {group_col!r} in v.columns}}",
450
+ f"if not _groupable_{suffix}:",
451
+ f" raise RuntimeError({f'group_by column {group_col!r} not found in any dataframe input of block {bid!r}'!r})",
452
+ f"_primary_port_{suffix} = next(iter(_groupable_{suffix}))",
453
+ f"_n_groups_{suffix} = _groupable_{suffix}[_primary_port_{suffix}].select(pl.col({group_col!r}).n_unique()).item() or 0",
454
+ f"if _n_groups_{suffix} == 0:",
455
+ f" raise RuntimeError({f'no groups found for group_by column {group_col!r}'!r})",
456
+ f"if _n_groups_{suffix} > 500:",
457
+ f" raise RuntimeError(f\"grouping by {group_col!r} would produce {{_n_groups_{suffix}}} groups, over the 500-group safety limit\")",
458
+ f"_partitions_{suffix} = {{"
459
+ f"port: {{k[0]: v for k, v in frame.partition_by({group_col!r}, as_dict=True).items()}} "
460
+ f"for port, frame in _groupable_{suffix}.items()}}",
461
+ f"_group_values_{suffix} = list(_partitions_{suffix}[_primary_port_{suffix}].keys())",
462
+ "",
463
+ f"def _task_{suffix}(_gval):",
464
+ f" _kwargs = dict(_df_kwargs_{suffix})",
465
+ f" _kwargs.update(_fixed_kwargs_{suffix})",
466
+ f" for _port, _parts in _partitions_{suffix}.items():",
467
+ " if _gval in _parts:",
468
+ " _kwargs[_port] = _parts[_gval]",
469
+ f" return {fn_name}(**_kwargs)",
470
+ "",
471
+ f"with ThreadPoolExecutor(max_workers=min({max_workers}, _n_groups_{suffix})) as _ex:",
472
+ f" _raw_{suffix} = dict(zip(_group_values_{suffix}, _ex.map(_task_{suffix}, _group_values_{suffix})))",
473
+ "",
474
+ f"_combined_{suffix} = _combine_group_results({group_col!r}, _group_values_{suffix}, _raw_{suffix}, {varlist!r})",
475
+ ]
476
+ if varlist:
477
+ lines.append(f"{', '.join(varlist)} = _combined_{suffix}" if len(varlist) > 1 else f"{varlist[0]} = _combined_{suffix}")
478
+ return lines
modelmaker/graph.py ADDED
@@ -0,0 +1,171 @@
1
+ from __future__ import annotations
2
+
3
+ import types
4
+ from dataclasses import dataclass, field
5
+ from typing import Any
6
+
7
+ import polars as pl
8
+
9
+ from .blocks.base import BLOCK_REGISTRY, BlockType, PortSpec
10
+
11
+
12
+ @dataclass
13
+ class Position:
14
+ x: float = 0
15
+ y: float = 0
16
+
17
+
18
+ @dataclass
19
+ class Lane:
20
+ name: str
21
+ order: int
22
+ # Pixel height of this lane's band on the canvas -- user-resizable by
23
+ # dragging its top/bottom border (see LaneBand.tsx). Lanes stack
24
+ # end-to-end by `order` with no gap, so lanes can never overlap by
25
+ # construction; resizing one only shifts the ones after it.
26
+ height: float = 260.0
27
+
28
+
29
+ @dataclass
30
+ class Wire:
31
+ id: str
32
+ from_block: str
33
+ from_port: str
34
+ to_block: str
35
+ to_port: str
36
+
37
+
38
+ @dataclass
39
+ class BlockInstance:
40
+ id: str
41
+ block_type: BlockType
42
+ category: str
43
+ name: str
44
+ lane: str | None
45
+ position: Position
46
+ code_version: int
47
+ params: dict[str, Any]
48
+ inputs: list[PortSpec]
49
+ outputs: list[PortSpec]
50
+ code_ref: str | None = None
51
+ code: str | None = None
52
+ # dict declaration for custom blocks (see metadata_transforms.resolve);
53
+ # None for registry blocks, whose transform lives in the registry.
54
+ metadata_transform: dict[str, Any] | None = None
55
+ # True for an AI-authored block with its own editable code/metadata_transform;
56
+ # False for a block backed by a fixed BLOCK_REGISTRY entry. Independent of
57
+ # block_type -- a custom block can be an input, standard, or output block.
58
+ is_custom: bool = False
59
+ # column name -> ColumnRole value (see packet.py), hand-tagged by the
60
+ # user (see session.set_column_role) and layered on top of whatever role
61
+ # this block's own metadata_transform already computed -- applies to
62
+ # that column on every output port it appears on. Included in
63
+ # Runner.compute_key's hash basis alongside params/code, so retagging a
64
+ # column invalidates this block and everything downstream the same way
65
+ # any other param change does; no separate staleness bookkeeping needed.
66
+ column_role_overrides: dict[str, str] = field(default_factory=dict)
67
+ # output port name -> user-given name for the data on that port (e.g.
68
+ # "raw applications"), settable by clicking that port's data in the UI
69
+ # (see PortInspector.tsx). Purely descriptive except in one place: the
70
+ # compiler uses it, when set, as the compiled variable name that holds
71
+ # this port's data instead of the auto-generated block-name/id scheme
72
+ # (see compiler.compile_graph) -- so naming your data also names it in
73
+ # the generated script.
74
+ port_names: dict[str, str] = field(default_factory=dict)
75
+ # When set, this block runs once per distinct value of this column
76
+ # (present in one of its dataframe inputs) instead of once overall --
77
+ # e.g. an auc_gini block with group_by="region" produces a Gini per
78
+ # region instead of one for the whole dataset. See Runner._run_grouped.
79
+ # None (the default) is plain, ungrouped execution.
80
+ group_by: str | None = None
81
+ # Cap on how many of this block's groups may run concurrently, when
82
+ # group_by is set. None means "use the runner's default" (see
83
+ # Runner.MAX_GROUP_WORKERS). Ignored for ungrouped blocks.
84
+ max_workers: int | None = None
85
+
86
+ def resolved_fn(self):
87
+ if self.is_custom:
88
+ return _compile_code_to_fn(self.code or "")
89
+ return BLOCK_REGISTRY[self.category].fn
90
+
91
+ def resolved_metadata_transform(self):
92
+ if self.is_custom:
93
+ return self.metadata_transform
94
+ return BLOCK_REGISTRY[self.category].metadata_transform
95
+
96
+
97
+ def _compile_code_to_fn(code: str):
98
+ # `pl` is available here to match the compiled script's module-level
99
+ # `import polars as pl` (see compiler.py) -- custom block code should be
100
+ # able to rely on it being in scope either way.
101
+ ns: dict[str, Any] = {"pl": pl}
102
+ exec(code, ns) # trusted, project-local code; matches the plan's no-sandboxing execution model
103
+ fns = [v for v in ns.values() if isinstance(v, types.FunctionType)]
104
+ if len(fns) != 1:
105
+ raise ValueError("custom block code must define exactly one top-level function")
106
+ return fns[0]
107
+
108
+
109
+ @dataclass
110
+ class Graph:
111
+ lanes: dict[str, Lane] = field(default_factory=dict)
112
+ blocks: dict[str, BlockInstance] = field(default_factory=dict)
113
+ wires: dict[str, Wire] = field(default_factory=dict)
114
+
115
+ def input_wires(self, block_id: str) -> dict[str, Wire]:
116
+ return {w.to_port: w for w in self.wires.values() if w.to_block == block_id}
117
+
118
+ def predecessors(self, block_id: str) -> set[str]:
119
+ return {w.from_block for w in self.wires.values() if w.to_block == block_id}
120
+
121
+ def successors(self, block_id: str) -> set[str]:
122
+ return {w.to_block for w in self.wires.values() if w.from_block == block_id}
123
+
124
+ def ancestors(self, block_ids: list[str]) -> set[str]:
125
+ seen: set[str] = set()
126
+ stack = list(block_ids)
127
+ while stack:
128
+ b = stack.pop()
129
+ if b in seen:
130
+ continue
131
+ seen.add(b)
132
+ stack.extend(self.predecessors(b))
133
+ return seen
134
+
135
+ def creates_cycle(self, from_block: str, to_block: str) -> bool:
136
+ """Would a wire from_block -> to_block create a cycle? True if
137
+ to_block is already an ancestor of from_block (including
138
+ from_block == to_block), i.e. a path to_block -> ... -> from_block
139
+ already exists, so adding from_block -> to_block would close a loop.
140
+ Checked before a wire is added -- see session.add_wire -- so the
141
+ graph a user can build is acyclic by construction and topo_order's
142
+ cycle branch is unreachable in normal use."""
143
+ return to_block in self.ancestors([from_block])
144
+
145
+ def topo_order(self) -> list[str]:
146
+ """Deterministic topological sort: ties broken by (lane order, canvas
147
+ position, block id) per plan section 7 — lanes play no role in
148
+ execution order itself, only in tie-breaking for stable output.
149
+ Blocks are ordered by y before x since flow runs top-to-bottom, to
150
+ match visual reading order for otherwise-unordered blocks."""
151
+ visited: set[str] = set()
152
+ order: list[str] = []
153
+
154
+ def sort_key(bid: str):
155
+ b = self.blocks[bid]
156
+ lane_order = self.lanes[b.lane].order if b.lane in self.lanes else 0
157
+ return (lane_order, b.position.y, b.position.x, bid)
158
+
159
+ def visit(bid: str, stack: frozenset[str]):
160
+ if bid in visited:
161
+ return
162
+ if bid in stack:
163
+ raise ValueError(f"cycle detected at block {bid}")
164
+ for pred in sorted(self.predecessors(bid), key=sort_key):
165
+ visit(pred, stack | {bid})
166
+ visited.add(bid)
167
+ order.append(bid)
168
+
169
+ for bid in sorted(self.blocks, key=sort_key):
170
+ visit(bid, frozenset())
171
+ return order
@@ -0,0 +1,28 @@
1
+ from .base import LLM_PROVIDER_REGISTRY, ColumnInfo, DraftContext, DraftResult, LLMProvider, get_provider, register_provider
2
+ from .claude_cli_provider import ClaudeCliProvider
3
+ from .gemini_provider import GeminiProvider
4
+ from .lmstudio_provider import LMStudioProvider
5
+ from .openai_provider import OpenAIProvider
6
+ from .stub_provider import StubProvider
7
+
8
+ __all__ = [
9
+ "LLM_PROVIDER_REGISTRY",
10
+ "ColumnInfo",
11
+ "DraftContext",
12
+ "DraftResult",
13
+ "LLMProvider",
14
+ "get_provider",
15
+ "register_provider",
16
+ "StubProvider",
17
+ "ClaudeCliProvider",
18
+ "LMStudioProvider",
19
+ "OpenAIProvider",
20
+ "GeminiProvider",
21
+ ]
22
+
23
+ try:
24
+ from .anthropic_provider import AnthropicProvider # noqa: F401
25
+
26
+ __all__.append("AnthropicProvider")
27
+ except ImportError:
28
+ pass # `anthropic` package not installed; the "anthropic" provider name is unavailable until it is.