quantology-modelmaker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelmaker/__init__.py +2 -0
- modelmaker/api.py +1073 -0
- modelmaker/blocks/__init__.py +0 -0
- modelmaker/blocks/base.py +78 -0
- modelmaker/blocks/library.py +327 -0
- modelmaker/blocks/modelling.py +312 -0
- modelmaker/blocks/stat_tests.py +133 -0
- modelmaker/cache.py +44 -0
- modelmaker/compiler.py +478 -0
- modelmaker/graph.py +171 -0
- modelmaker/llm/__init__.py +28 -0
- modelmaker/llm/anthropic_provider.py +79 -0
- modelmaker/llm/base.py +85 -0
- modelmaker/llm/claude_cli_provider.py +90 -0
- modelmaker/llm/gemini_provider.py +138 -0
- modelmaker/llm/lmstudio_provider.py +140 -0
- modelmaker/llm/openai_provider.py +138 -0
- modelmaker/llm/prompts.py +189 -0
- modelmaker/llm/settings.py +47 -0
- modelmaker/llm/stub_provider.py +29 -0
- modelmaker/metadata_transforms.py +109 -0
- modelmaker/packet.py +111 -0
- modelmaker/project.py +129 -0
- modelmaker/run_worker.py +143 -0
- modelmaker/runner.py +1313 -0
- modelmaker/session.py +400 -0
- modelmaker/static/assets/index-B9cHjTMW.css +1 -0
- modelmaker/static/assets/index-EvjTc-1k.js +34 -0
- modelmaker/static/brand-quantology/favicon.svg +4 -0
- modelmaker/static/brand-quantology/icon-mark-light.svg +13 -0
- modelmaker/static/icons.svg +24 -0
- modelmaker/static/index.html +20 -0
- modelmaker/tui/__init__.py +0 -0
- modelmaker/tui/app.py +785 -0
- modelmaker/tui/canvas.py +388 -0
- modelmaker/tui/charts.py +84 -0
- modelmaker/tui/client.py +270 -0
- modelmaker/tui/inspector.py +285 -0
- modelmaker/tui/paramspecs.py +128 -0
- modelmaker/tui/preview.py +71 -0
- modelmaker/tui/screens.py +182 -0
- modelmaker/util.py +44 -0
- quantology_modelmaker-0.1.0.dist-info/METADATA +247 -0
- quantology_modelmaker-0.1.0.dist-info/RECORD +49 -0
- quantology_modelmaker-0.1.0.dist-info/WHEEL +5 -0
- quantology_modelmaker-0.1.0.dist-info/entry_points.txt +3 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/LICENSE +204 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/NOTICE +12 -0
- quantology_modelmaker-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from .base import ColumnInfo, DraftContext
|
|
7
|
+
|
|
8
|
+
JSON_ONLY_INSTRUCTIONS = """
|
|
9
|
+
Respond with ONLY a single JSON object -- no markdown code fences, no prose \
|
|
10
|
+
before or after it. It must have exactly these keys:
|
|
11
|
+
{
|
|
12
|
+
"code": "<the complete function definition as a string>",
|
|
13
|
+
"metadata_transform": {
|
|
14
|
+
"kind": "passthrough" | "narrow" | "declared",
|
|
15
|
+
"base": "<input port name, only when kind is 'declared', else null>",
|
|
16
|
+
"drops": ["<column name>", ...],
|
|
17
|
+
"adds": [{"name": "...", "dtype": "...", "role": "..."}, ...]
|
|
18
|
+
},
|
|
19
|
+
"params": {"<param name>": <default value>, ...},
|
|
20
|
+
"explanation": "<one or two sentences>"
|
|
21
|
+
}
|
|
22
|
+
Omit "drops"/"adds"/"base" (send empty list / null) when kind is not \
|
|
23
|
+
"declared". The response must be valid JSON parseable by a standard parser."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def extract_json_response(text: str) -> dict:
|
|
27
|
+
"""Pull a JSON object out of a model's raw text reply, tolerating markdown
|
|
28
|
+
code fences or stray prose around the object (small/local models in
|
|
29
|
+
particular tend not to follow "JSON only" instructions exactly)."""
|
|
30
|
+
text = text.strip()
|
|
31
|
+
try:
|
|
32
|
+
return json.loads(text)
|
|
33
|
+
except json.JSONDecodeError:
|
|
34
|
+
pass
|
|
35
|
+
fenced = re.search(r"```(?:json)?\s*(\{.*\})\s*```", text, re.DOTALL)
|
|
36
|
+
if fenced:
|
|
37
|
+
return json.loads(fenced.group(1))
|
|
38
|
+
brace = re.search(r"\{.*\}", text, re.DOTALL)
|
|
39
|
+
if brace:
|
|
40
|
+
return json.loads(brace.group(0))
|
|
41
|
+
raise ValueError(f"model did not return parseable JSON: {text[:500]!r}")
|
|
42
|
+
|
|
43
|
+
CONTRACT = """You draft a single block's implementation for a visual, \
|
|
44
|
+
Polars-based data pipeline tool. Follow the contract exactly:
|
|
45
|
+
|
|
46
|
+
- Write exactly one top-level Python function named as instructed. It takes \
|
|
47
|
+
`df: pl.DataFrame` as its first parameter, plus any additional parameters you \
|
|
48
|
+
introduce for configurable values (each with a sensible default), and returns \
|
|
49
|
+
a single `pl.DataFrame`.
|
|
50
|
+
- Use only the `polars` library (imported as `pl`). Never use pandas.
|
|
51
|
+
- The function body is inlined verbatim into a generated script and also run \
|
|
52
|
+
directly by the engine -- do not reference anything outside the function \
|
|
53
|
+
(no globals, no I/O, no network calls). It must be a pure function of its \
|
|
54
|
+
arguments.
|
|
55
|
+
- Only reference columns that are listed in the provided input schema. If the \
|
|
56
|
+
instruction implies a new column, add it with `.with_columns(...)`.
|
|
57
|
+
- Never hardcode an *existing* input column's name as a string literal in the \
|
|
58
|
+
function body. For each existing column your logic reads, add a `str` \
|
|
59
|
+
parameter named `<descriptive_name>_col`, defaulting to that column's real \
|
|
60
|
+
name from the input schema, and reference the parameter (e.g. \
|
|
61
|
+
`pl.col(income_col)`) instead of the literal -- this lets the tool re-point \
|
|
62
|
+
the block at a differently-named upstream column later without touching the \
|
|
63
|
+
code. Columns you are newly creating still get fixed, literal names via \
|
|
64
|
+
`.alias(...)`; only pre-existing columns you read need a `_col` parameter.
|
|
65
|
+
- Also produce a `metadata_transform` describing how the function changes the \
|
|
66
|
+
column set, so the tool can track column roles/types without re-executing \
|
|
67
|
+
your code:
|
|
68
|
+
- kind="passthrough": output columns are exactly the input columns, \
|
|
69
|
+
unchanged (dtype may differ; the tool re-infers dtype either way).
|
|
70
|
+
- kind="narrow": output columns are a subset of the input columns (e.g. a \
|
|
71
|
+
select/filter that drops columns).
|
|
72
|
+
- kind="declared": output adds and/or drops specific columns. Set `base` \
|
|
73
|
+
to the input port name ("df"), list any dropped column names in `drops`, and \
|
|
74
|
+
describe every added column in `adds` (name, a short polars dtype string like \
|
|
75
|
+
"Int64"/"Float64"/"Boolean"/"String"/"Date", and a role -- one of id, target, \
|
|
76
|
+
weight, feature, date, segment, excluded, unassigned).
|
|
77
|
+
- If `params` values are provided as suggested defaults, they must match the \
|
|
78
|
+
extra parameter names in your function signature exactly.
|
|
79
|
+
- If existing code and an error are provided, this is a bug-fix request: fix \
|
|
80
|
+
the described error while preserving the function's original intent as \
|
|
81
|
+
closely as possible. Do not rewrite unrelated behavior.
|
|
82
|
+
- Keep `explanation` to one or two plain-English sentences describing what \
|
|
83
|
+
the function does or what you changed and why."""
|
|
84
|
+
|
|
85
|
+
CONTRACT_PARAMS_ONLY = """You are configuring one block in a visual, \
|
|
86
|
+
Polars-based data pipeline tool. This block's Python function is fixed -- \
|
|
87
|
+
you cannot change its code, only choose values for the parameters it \
|
|
88
|
+
already accepts. Follow the contract exactly:
|
|
89
|
+
|
|
90
|
+
- The fixed function is given below (read-only). Read its signature and \
|
|
91
|
+
docstring to see exactly which parameters exist, their types, and their \
|
|
92
|
+
defaults.
|
|
93
|
+
- Respond with a `params` object containing values for whichever of those \
|
|
94
|
+
parameters the instruction implies should change. Only use keys that are \
|
|
95
|
+
real parameter names of the function -- never invent new ones, and never \
|
|
96
|
+
include the dataframe argument(s) themselves.
|
|
97
|
+
- Only reference columns listed in the provided input schema.
|
|
98
|
+
- Leave `code` as an empty string and `metadata_transform` as \
|
|
99
|
+
`{"kind": "passthrough"}` -- both are ignored for this block.
|
|
100
|
+
- Keep `explanation` to one sentence describing the values you chose and \
|
|
101
|
+
why."""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
POLARS_REFERENCE = """
|
|
105
|
+
Condensed Polars reference -- consult this instead of relying on possibly \
|
|
106
|
+
stale training knowledge of the API, especially for smaller/local models:
|
|
107
|
+
|
|
108
|
+
All of `.select()`/`.filter()`/`.with_columns()`/`.group_by().agg()` are \
|
|
109
|
+
expression-based -- write `pl.col("name")`, not `df["name"]`. There is no \
|
|
110
|
+
pandas-style label indexing (`.loc`/`.iloc`/`df["col"] = ...`).
|
|
111
|
+
|
|
112
|
+
- Select/rename: `df.select(pl.col("a"), pl.col("b").alias("b2"))`
|
|
113
|
+
- Add/replace a column: `df.with_columns((pl.col("a") * 2).alias("a_doubled"))`
|
|
114
|
+
- Filter rows: `df.filter(pl.col("score") > 700)`
|
|
115
|
+
- Conditional column: `df.with_columns(pl.when(pl.col("x") > 0).then(pl.lit("pos")).otherwise(pl.lit("neg")).alias("sign"))`
|
|
116
|
+
- Group + aggregate: `df.group_by("segment").agg(pl.col("amount").sum().alias("total"))`
|
|
117
|
+
- Join: `df.join(other, on="id", how="left")`
|
|
118
|
+
- Sort: `df.sort("date", descending=True)`
|
|
119
|
+
- Cast dtype: `pl.col("x").cast(pl.Float64)`
|
|
120
|
+
- String ops: `pl.col("name").str.to_lowercase()`, `.str.contains("x")`, `.str.replace_all(a, b)`
|
|
121
|
+
- Date ops: `pl.col("d").dt.year()`, `.dt.strftime("%Y-%m")`
|
|
122
|
+
- Null handling: `pl.col("x").fill_null(0)`, `.is_null()`, `.drop_nulls()`
|
|
123
|
+
- Row count / unique: `df.height`, `df.select(pl.col("x").n_unique())`
|
|
124
|
+
|
|
125
|
+
Common mistakes to avoid:
|
|
126
|
+
- `.alias()` is required to name an expression's output column; without it \
|
|
127
|
+
the output keeps the input expression's original column name.
|
|
128
|
+
- `pl.when/then/otherwise` is the ternary/`np.where` equivalent -- there is \
|
|
129
|
+
no `np.where` and no `.apply(lambda ...)` in idiomatic polars code.
|
|
130
|
+
- Boolean masks combine with `&`/`|` (never `and`/`or`), each operand \
|
|
131
|
+
parenthesized: `df.filter((pl.col("a") > 1) & (pl.col("b") < 5))`.
|
|
132
|
+
|
|
133
|
+
Worked example -- instruction: "flag rows where income > 50000 as high_income"
|
|
134
|
+
```python
|
|
135
|
+
def bucket_income(df: pl.DataFrame, income_col: str = "income") -> pl.DataFrame:
|
|
136
|
+
return df.with_columns(
|
|
137
|
+
(pl.col(income_col) > 50000).alias("high_income")
|
|
138
|
+
)
|
|
139
|
+
```
|
|
140
|
+
`income` is an existing column being read, so it becomes the `income_col` \
|
|
141
|
+
parameter (defaulted to its real name) instead of a literal; `high_income` is \
|
|
142
|
+
a new column, so it keeps a fixed, literal name via `.alias(...)`. `params` \
|
|
143
|
+
would be `{"income_col": "income"}`, and this adds one boolean column, so \
|
|
144
|
+
`metadata_transform` would be \
|
|
145
|
+
`{"kind": "declared", "base": "df", "drops": [], "adds": [{"name": "high_income", "dtype": "Boolean", "role": "feature"}]}`.
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def contract_for(mode: str, include_reference: bool = False) -> str:
|
|
150
|
+
if mode == "params_only":
|
|
151
|
+
return CONTRACT_PARAMS_ONLY
|
|
152
|
+
return CONTRACT + ("\n" + POLARS_REFERENCE if include_reference else "")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def format_columns(columns: list[ColumnInfo]) -> str:
|
|
156
|
+
if not columns:
|
|
157
|
+
return " (unknown -- this input hasn't been run yet, so its columns aren't available)"
|
|
158
|
+
return "\n".join(f" - {c.name}: {c.dtype}, role={c.role}" for c in columns)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def build_user_prompt(ctx: DraftContext) -> str:
|
|
162
|
+
parts = [f"Function name: {ctx.function_name}"]
|
|
163
|
+
|
|
164
|
+
if ctx.fixed_source:
|
|
165
|
+
parts += [
|
|
166
|
+
"",
|
|
167
|
+
"Fixed function source (read-only -- you are only choosing parameter values, not writing code):",
|
|
168
|
+
"```python",
|
|
169
|
+
ctx.fixed_source,
|
|
170
|
+
"```",
|
|
171
|
+
]
|
|
172
|
+
|
|
173
|
+
parts += ["", "Input schema:"]
|
|
174
|
+
for port, cols in (ctx.input_ports or {}).items():
|
|
175
|
+
parts.append(f" Port `{port}`:")
|
|
176
|
+
parts.append(format_columns(cols))
|
|
177
|
+
|
|
178
|
+
if ctx.param_names:
|
|
179
|
+
parts += ["", "Current parameter names: " + ", ".join(ctx.param_names)]
|
|
180
|
+
|
|
181
|
+
if ctx.existing_code:
|
|
182
|
+
parts += ["", "Existing code:", "```python", ctx.existing_code, "```"]
|
|
183
|
+
|
|
184
|
+
if ctx.error:
|
|
185
|
+
parts += ["", "This code just failed with the following error -- fix it:", ctx.error, "", f"Additional instruction: {ctx.instruction}"]
|
|
186
|
+
else:
|
|
187
|
+
parts += ["", f"Instruction: {ctx.instruction}"]
|
|
188
|
+
|
|
189
|
+
return "\n".join(parts)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class LLMSettingsStore:
|
|
9
|
+
"""UI-editable LLM settings for the running server, kept in memory only
|
|
10
|
+
(like ProjectSession -- nothing here persists across a restart). Every
|
|
11
|
+
field defaults to "unset" (None/empty): until the Settings panel changes
|
|
12
|
+
something, each provider falls back to its own constructor defaults
|
|
13
|
+
(env vars, then hardcoded constants), exactly as before this store
|
|
14
|
+
existed. Setting a field here overrides that provider's default until
|
|
15
|
+
changed again or the server restarts.
|
|
16
|
+
|
|
17
|
+
This includes API keys (per_provider[name]["api_key"]): they live only
|
|
18
|
+
in this process's memory, are never written to disk, and the API layer
|
|
19
|
+
(see api.py's _effective_llm_settings) never echoes a raw key value back
|
|
20
|
+
to the client -- only whether one is currently set and where it came
|
|
21
|
+
from (an explicit override here vs. an environment variable)."""
|
|
22
|
+
|
|
23
|
+
active_provider: str | None = None
|
|
24
|
+
per_provider: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
25
|
+
# Global include-Polars-reference toggle (see DraftContext.include_reference),
|
|
26
|
+
# applied across every provider's draft calls. None = each provider's own default.
|
|
27
|
+
include_reference: bool | None = None
|
|
28
|
+
|
|
29
|
+
def for_provider(self, name: str) -> dict[str, Any]:
|
|
30
|
+
return self.per_provider.get(name, {})
|
|
31
|
+
|
|
32
|
+
def update(self, active_provider: str | None, include_reference: bool | None, settings: dict[str, dict[str, Any]] | None) -> None:
|
|
33
|
+
if active_provider is not None:
|
|
34
|
+
self.active_provider = active_provider
|
|
35
|
+
if include_reference is not None:
|
|
36
|
+
self.include_reference = include_reference
|
|
37
|
+
for name, values in (settings or {}).items():
|
|
38
|
+
slot = self.per_provider.setdefault(name, {})
|
|
39
|
+
for key, value in values.items():
|
|
40
|
+
# An explicit null clears a previously-set override (e.g. an
|
|
41
|
+
# emptied text field, or "Remove key") so the provider falls
|
|
42
|
+
# back to its own default (env var / hardcoded constant)
|
|
43
|
+
# instead of being stuck with a stale value.
|
|
44
|
+
if value is None:
|
|
45
|
+
slot.pop(key, None)
|
|
46
|
+
else:
|
|
47
|
+
slot[key] = value
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from .base import DraftContext, DraftResult, LLMProvider, register_provider
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_provider("stub")
|
|
7
|
+
class StubProvider(LLMProvider):
|
|
8
|
+
"""Deterministic, no-network provider. Used by default in tests and as a
|
|
9
|
+
template for adding a real second provider alongside AnthropicProvider --
|
|
10
|
+
register a class with @register_provider("name") and point
|
|
11
|
+
MODELMAKER_LLM_PROVIDER at it."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, model: str | None = None):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
def draft(self, ctx: DraftContext) -> DraftResult:
|
|
17
|
+
if ctx.mode == "params_only":
|
|
18
|
+
explanation = (
|
|
19
|
+
"Stub provider: no LLM was called, so no parameter values were suggested -- "
|
|
20
|
+
f"replace with a real provider to act on: {ctx.instruction!r}"
|
|
21
|
+
)
|
|
22
|
+
return DraftResult(explanation=explanation)
|
|
23
|
+
|
|
24
|
+
code = f"def {ctx.function_name}(df):\n return df\n"
|
|
25
|
+
explanation = (
|
|
26
|
+
"Stub provider: no LLM was called. Returned the input unchanged -- "
|
|
27
|
+
f"replace with a real provider to act on: {ctx.instruction!r}"
|
|
28
|
+
)
|
|
29
|
+
return DraftResult(code=code, metadata_transform={"kind": "passthrough"}, explanation=explanation)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable
|
|
4
|
+
|
|
5
|
+
import polars as pl
|
|
6
|
+
|
|
7
|
+
from .packet import ColumnMeta, ColumnRole
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def infer_dtypes(
|
|
11
|
+
input_metas: dict[str, dict[str, ColumnMeta]],
|
|
12
|
+
outputs: dict[str, pl.DataFrame],
|
|
13
|
+
params: dict[str, Any],
|
|
14
|
+
) -> dict[str, dict[str, ColumnMeta]]:
|
|
15
|
+
"""Used by input blocks: no upstream metadata to inherit, so build fresh
|
|
16
|
+
ColumnMeta from dtype alone; role stays unassigned until tagged."""
|
|
17
|
+
return {
|
|
18
|
+
port: {name: ColumnMeta(dtype=str(df.schema[name])) for name in df.columns}
|
|
19
|
+
for port, df in outputs.items()
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def passthrough(
|
|
24
|
+
input_metas: dict[str, dict[str, ColumnMeta]],
|
|
25
|
+
outputs: dict[str, pl.DataFrame],
|
|
26
|
+
params: dict[str, Any],
|
|
27
|
+
) -> dict[str, dict[str, ColumnMeta]]:
|
|
28
|
+
"""Single input, single output, columns unchanged (e.g. filter)."""
|
|
29
|
+
(in_meta,) = input_metas.values()
|
|
30
|
+
result: dict[str, dict[str, ColumnMeta]] = {}
|
|
31
|
+
for port, df in outputs.items():
|
|
32
|
+
result[port] = {
|
|
33
|
+
name: in_meta[name] if name in in_meta else ColumnMeta(dtype=str(df.schema[name]))
|
|
34
|
+
for name in df.columns
|
|
35
|
+
}
|
|
36
|
+
return result
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def narrow_from_single_input(
|
|
40
|
+
input_metas: dict[str, dict[str, ColumnMeta]],
|
|
41
|
+
outputs: dict[str, pl.DataFrame],
|
|
42
|
+
params: dict[str, Any],
|
|
43
|
+
) -> dict[str, dict[str, ColumnMeta]]:
|
|
44
|
+
"""Single input, output columns are a subset of it (e.g. select)."""
|
|
45
|
+
(in_meta,) = input_metas.values()
|
|
46
|
+
result: dict[str, dict[str, ColumnMeta]] = {}
|
|
47
|
+
for port, df in outputs.items():
|
|
48
|
+
result[port] = {name: in_meta[name] for name in df.columns if name in in_meta}
|
|
49
|
+
return result
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def broadcast_to_all_outputs(
|
|
53
|
+
input_metas: dict[str, dict[str, ColumnMeta]],
|
|
54
|
+
outputs: dict[str, pl.DataFrame],
|
|
55
|
+
params: dict[str, Any],
|
|
56
|
+
) -> dict[str, dict[str, ColumnMeta]]:
|
|
57
|
+
"""Single input, metadata unchanged, fanned out to every output port
|
|
58
|
+
(e.g. train/test split: both sides keep the parent's column roles)."""
|
|
59
|
+
(in_meta,) = input_metas.values()
|
|
60
|
+
return {port: dict(in_meta) for port in outputs}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def declared(spec: dict[str, Any]) -> Callable[..., dict[str, dict[str, ColumnMeta]]]:
|
|
64
|
+
"""Build a metadata transform from a JSON-serializable declaration, the
|
|
65
|
+
form used by custom/LLM-authored blocks in the project file:
|
|
66
|
+
{"kind": "declared", "base": "in", "drops": [...], "adds": [...]}
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def _fn(
|
|
70
|
+
input_metas: dict[str, dict[str, ColumnMeta]],
|
|
71
|
+
outputs: dict[str, pl.DataFrame],
|
|
72
|
+
params: dict[str, Any],
|
|
73
|
+
) -> dict[str, dict[str, ColumnMeta]]:
|
|
74
|
+
base_port = spec.get("base")
|
|
75
|
+
base: dict[str, ColumnMeta] = dict(input_metas[base_port]) if base_port else {}
|
|
76
|
+
for col in spec.get("drops", []):
|
|
77
|
+
base.pop(col, None)
|
|
78
|
+
for add in spec.get("adds", []):
|
|
79
|
+
base[add["name"]] = ColumnMeta(
|
|
80
|
+
dtype=add.get("dtype", "unknown"),
|
|
81
|
+
role=ColumnRole(add.get("role", "unassigned")),
|
|
82
|
+
description=add.get("description"),
|
|
83
|
+
)
|
|
84
|
+
result: dict[str, dict[str, ColumnMeta]] = {}
|
|
85
|
+
for port, df in outputs.items():
|
|
86
|
+
result[port] = {
|
|
87
|
+
name: base[name] if name in base else ColumnMeta(dtype=str(df.schema[name]))
|
|
88
|
+
for name in df.columns
|
|
89
|
+
}
|
|
90
|
+
return result
|
|
91
|
+
|
|
92
|
+
return _fn
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def resolve_metadata_transform(transform: Any) -> Callable[..., dict[str, dict[str, ColumnMeta]]]:
|
|
96
|
+
if callable(transform):
|
|
97
|
+
return transform
|
|
98
|
+
kind = transform.get("kind")
|
|
99
|
+
if kind == "infer_dtypes":
|
|
100
|
+
return infer_dtypes
|
|
101
|
+
if kind == "passthrough":
|
|
102
|
+
return passthrough
|
|
103
|
+
if kind == "narrow":
|
|
104
|
+
return narrow_from_single_input
|
|
105
|
+
if kind == "broadcast":
|
|
106
|
+
return broadcast_to_all_outputs
|
|
107
|
+
if kind == "declared":
|
|
108
|
+
return declared(transform)
|
|
109
|
+
raise ValueError(f"unknown metadata_transform kind: {kind!r}")
|
modelmaker/packet.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field, replace
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import polars as pl
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ColumnRole(str, Enum):
|
|
11
|
+
ID = "id"
|
|
12
|
+
TARGET = "target"
|
|
13
|
+
PREDICTED = "predicted"
|
|
14
|
+
WEIGHT = "weight"
|
|
15
|
+
FEATURE = "feature"
|
|
16
|
+
DATE = "date"
|
|
17
|
+
SEGMENT = "segment"
|
|
18
|
+
EXCLUDED = "excluded"
|
|
19
|
+
UNASSIGNED = "unassigned"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Roles that may sit on at most one column within any single schema (a
|
|
23
|
+
# block's own output, or wherever schemas get merged, e.g. join) --
|
|
24
|
+
# everything else (feature/segment/excluded/unassigned) is fine on any
|
|
25
|
+
# number of columns at once.
|
|
26
|
+
UNIQUE_ROLES = frozenset({ColumnRole.ID, ColumnRole.TARGET, ColumnRole.PREDICTED, ColumnRole.WEIGHT, ColumnRole.DATE})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ColumnMeta:
|
|
31
|
+
dtype: str
|
|
32
|
+
role: ColumnRole = ColumnRole.UNASSIGNED
|
|
33
|
+
description: str | None = None
|
|
34
|
+
tags: list[str] = field(default_factory=list)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class ColumnStats:
|
|
39
|
+
count: int
|
|
40
|
+
null_count: int
|
|
41
|
+
n_unique: int | None = None
|
|
42
|
+
mean: float | None = None
|
|
43
|
+
std: float | None = None
|
|
44
|
+
min: Any = None
|
|
45
|
+
max: Any = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class DataFramePacket:
|
|
50
|
+
data: pl.DataFrame
|
|
51
|
+
schema_meta: dict[str, ColumnMeta]
|
|
52
|
+
lineage: list[str] = field(default_factory=list)
|
|
53
|
+
summary: dict[str, ColumnStats] | None = None
|
|
54
|
+
|
|
55
|
+
def with_lineage(self, block_id: str) -> DataFramePacket:
|
|
56
|
+
return replace(self, lineage=[*self.lineage, block_id], summary=None)
|
|
57
|
+
|
|
58
|
+
def compute_summary(self, force: bool = False) -> DataFramePacket:
|
|
59
|
+
"""Populate per-column summary stats on demand (not eager on every wire)."""
|
|
60
|
+
if self.summary is not None and not force:
|
|
61
|
+
return self
|
|
62
|
+
summary: dict[str, ColumnStats] = {}
|
|
63
|
+
for name in self.data.columns:
|
|
64
|
+
s = self.data[name]
|
|
65
|
+
is_numeric = s.dtype.is_numeric()
|
|
66
|
+
n = s.len()
|
|
67
|
+
summary[name] = ColumnStats(
|
|
68
|
+
count=n,
|
|
69
|
+
null_count=s.null_count(),
|
|
70
|
+
n_unique=s.n_unique(),
|
|
71
|
+
mean=float(s.mean()) if is_numeric and n else None,
|
|
72
|
+
std=float(s.std()) if is_numeric and n and n > 1 else None,
|
|
73
|
+
min=s.min(),
|
|
74
|
+
max=s.max(),
|
|
75
|
+
)
|
|
76
|
+
return replace(self, summary=summary)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def find_duplicate_unique_role(schema_meta: dict[str, ColumnMeta]) -> tuple[ColumnRole, list[str]] | None:
|
|
80
|
+
"""A unique role (see UNIQUE_ROLES) sitting on more than one column
|
|
81
|
+
within one schema is a real conflict, not a preference -- e.g. a join
|
|
82
|
+
silently combining two independently-tagged target columns. Returns the
|
|
83
|
+
offending role and its (sorted) column names, or None if the schema is
|
|
84
|
+
clean. Called both when a role is hand-tagged (immediate feedback) and
|
|
85
|
+
after every block run (the backstop that catches a merge-time collision
|
|
86
|
+
neither side could have seen on its own)."""
|
|
87
|
+
by_role: dict[ColumnRole, list[str]] = {}
|
|
88
|
+
for name, meta in schema_meta.items():
|
|
89
|
+
if meta.role in UNIQUE_ROLES:
|
|
90
|
+
by_role.setdefault(meta.role, []).append(name)
|
|
91
|
+
for role, cols in by_role.items():
|
|
92
|
+
if len(cols) > 1:
|
|
93
|
+
return role, sorted(cols)
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def resolve_role_column(schema_metas: list[dict[str, ColumnMeta]], role: ColumnRole) -> str | None:
|
|
98
|
+
"""The column tagged the given role among one or more input schemas, for
|
|
99
|
+
auto-filling a block's role-named param (e.g. target/target_col for
|
|
100
|
+
role=TARGET, score_col/predicted_col for role=PREDICTED) when left unset
|
|
101
|
+
(see util.find_role_param). None if no column is tagged, or if more than
|
|
102
|
+
one distinctly-named column is (ambiguous -- caller leaves the param
|
|
103
|
+
unset rather than guessing, which surfaces as a normal missing-argument
|
|
104
|
+
error)."""
|
|
105
|
+
candidates = {name for meta in schema_metas for name, m in meta.items() if m.role == role}
|
|
106
|
+
return next(iter(candidates)) if len(candidates) == 1 else None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def resolve_target_column(schema_metas: list[dict[str, ColumnMeta]]) -> str | None:
|
|
110
|
+
"""The column tagged role=target -- see resolve_role_column."""
|
|
111
|
+
return resolve_role_column(schema_metas, ColumnRole.TARGET)
|
modelmaker/project.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import asdict
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from .blocks.base import PortSpec
|
|
9
|
+
from .graph import BlockInstance, Graph, Lane, Position, Wire
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def graph_from_dict(data: dict[str, Any], project_dir: Path | None = None) -> Graph:
|
|
13
|
+
"""Rebuild a Graph from its serialized form. A custom block's code comes
|
|
14
|
+
from an inline "code" key when present (how in-memory snapshots carry it
|
|
15
|
+
-- see graph_to_dict), else from the sidecar file named by "code_ref",
|
|
16
|
+
resolved against project_dir (how a saved project carries it)."""
|
|
17
|
+
lanes = {
|
|
18
|
+
lid: Lane(name=lane["name"], order=lane["order"], height=lane.get("height", 260.0))
|
|
19
|
+
for lid, lane in data.get("lanes", {}).items()
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
blocks: dict[str, BlockInstance] = {}
|
|
23
|
+
for bid, b in data.get("blocks", {}).items():
|
|
24
|
+
code = b.get("code")
|
|
25
|
+
if code is None and b.get("code_ref") and project_dir is not None:
|
|
26
|
+
code = (project_dir / b["code_ref"]).read_text(encoding="utf-8")
|
|
27
|
+
pos = b.get("position", {"x": 0, "y": 0})
|
|
28
|
+
blocks[bid] = BlockInstance(
|
|
29
|
+
id=bid,
|
|
30
|
+
block_type=b["block_type"],
|
|
31
|
+
category=b["category"],
|
|
32
|
+
name=b["name"],
|
|
33
|
+
lane=b.get("lane"),
|
|
34
|
+
position=Position(x=pos["x"], y=pos["y"]),
|
|
35
|
+
code_version=b.get("code_version", 1),
|
|
36
|
+
params=b.get("params", {}),
|
|
37
|
+
inputs=[PortSpec(**p) for p in b["ports"]["inputs"]],
|
|
38
|
+
outputs=[PortSpec(**p) for p in b["ports"]["outputs"]],
|
|
39
|
+
code_ref=b.get("code_ref"),
|
|
40
|
+
code=code,
|
|
41
|
+
metadata_transform=b.get("metadata_transform"),
|
|
42
|
+
is_custom=b.get("is_custom", False),
|
|
43
|
+
column_role_overrides=b.get("column_role_overrides", {}),
|
|
44
|
+
port_names=b.get("port_names", {}),
|
|
45
|
+
group_by=b.get("group_by"),
|
|
46
|
+
max_workers=b.get("max_workers"),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
wires = {
|
|
50
|
+
wid: Wire(
|
|
51
|
+
id=wid,
|
|
52
|
+
from_block=w["from"]["block"],
|
|
53
|
+
from_port=w["from"]["port"],
|
|
54
|
+
to_block=w["to"]["block"],
|
|
55
|
+
to_port=w["to"]["port"],
|
|
56
|
+
)
|
|
57
|
+
for wid, w in data.get("wires", {}).items()
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return Graph(lanes=lanes, blocks=blocks, wires=wires)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def graph_to_dict(graph: Graph, project_name: str = "project", project_dir: Path | None = None) -> dict[str, Any]:
|
|
64
|
+
"""Serialize a Graph. With project_dir given, a custom block's code is
|
|
65
|
+
written to a sidecar .py file and referenced by "code_ref" (keeps git
|
|
66
|
+
diffs line-level -- see the plan's section 8.1); without one, the code
|
|
67
|
+
rides inline under "code" instead, for snapshots that never touch disk."""
|
|
68
|
+
blocks_out: dict[str, Any] = {}
|
|
69
|
+
for bid, b in sorted(graph.blocks.items()):
|
|
70
|
+
code_ref = b.code_ref
|
|
71
|
+
inline_code: str | None = None
|
|
72
|
+
if b.is_custom and b.code is not None:
|
|
73
|
+
if project_dir is not None:
|
|
74
|
+
code_ref = code_ref or f"blocks/{bid}.py"
|
|
75
|
+
sidecar = project_dir / code_ref
|
|
76
|
+
sidecar.parent.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
sidecar.write_text(b.code, encoding="utf-8")
|
|
78
|
+
else:
|
|
79
|
+
inline_code = b.code
|
|
80
|
+
entry = {
|
|
81
|
+
"block_type": b.block_type,
|
|
82
|
+
"is_custom": b.is_custom,
|
|
83
|
+
"category": b.category,
|
|
84
|
+
"name": b.name,
|
|
85
|
+
"lane": b.lane,
|
|
86
|
+
"position": {"x": b.position.x, "y": b.position.y},
|
|
87
|
+
"code_version": b.code_version,
|
|
88
|
+
"params": b.params,
|
|
89
|
+
"code_ref": code_ref,
|
|
90
|
+
"metadata_transform": b.metadata_transform if b.is_custom else None,
|
|
91
|
+
"column_role_overrides": b.column_role_overrides,
|
|
92
|
+
"port_names": b.port_names,
|
|
93
|
+
"group_by": b.group_by,
|
|
94
|
+
"max_workers": b.max_workers,
|
|
95
|
+
"ports": {
|
|
96
|
+
"inputs": [asdict(p) for p in b.inputs],
|
|
97
|
+
"outputs": [asdict(p) for p in b.outputs],
|
|
98
|
+
},
|
|
99
|
+
}
|
|
100
|
+
if inline_code is not None:
|
|
101
|
+
entry["code"] = inline_code
|
|
102
|
+
blocks_out[bid] = entry
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
"modelmaker_version": 1,
|
|
106
|
+
"project_name": project_name,
|
|
107
|
+
"lanes": {
|
|
108
|
+
lid: {"name": lane.name, "order": lane.order, "height": lane.height}
|
|
109
|
+
for lid, lane in sorted(graph.lanes.items(), key=lambda kv: kv[1].order)
|
|
110
|
+
},
|
|
111
|
+
"blocks": blocks_out,
|
|
112
|
+
"wires": {
|
|
113
|
+
wid: {
|
|
114
|
+
"from": {"block": w.from_block, "port": w.from_port},
|
|
115
|
+
"to": {"block": w.to_block, "port": w.to_port},
|
|
116
|
+
}
|
|
117
|
+
for wid, w in sorted(graph.wires.items())
|
|
118
|
+
},
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def load_project(path: Path) -> Graph:
|
|
123
|
+
return graph_from_dict(json.loads(path.read_text(encoding="utf-8")), project_dir=path.parent)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def save_project(graph: Graph, path: Path, project_name: str = "project") -> None:
|
|
127
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
128
|
+
data = graph_to_dict(graph, project_name=project_name, project_dir=path.parent)
|
|
129
|
+
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|