trail-lang 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trail/__init__.py +1 -0
- trail/ast.py +173 -0
- trail/catalog.py +245 -0
- trail/cli.py +102 -0
- trail/compiler.py +140 -0
- trail/config.py +64 -0
- trail/deps.py +80 -0
- trail/fixtures.py +55 -0
- trail/grammar.lark +92 -0
- trail/library.py +20 -0
- trail/macro.py +102 -0
- trail/ops.py +133 -0
- trail/parser.py +211 -0
- trail/pipeline.py +24 -0
- trail/py.typed +0 -0
- trail/registry.py +50 -0
- trail/schema.py +54 -0
- trail/source.py +118 -0
- trail/sources.py +135 -0
- trail/stdlib/__init__.py +0 -0
- trail/stdlib/calculus.trail +12 -0
- trail/stdlib/core.trail +13 -0
- trail/stdlib/factor.trail +13 -0
- trail/stdlib/geometry.trail +5 -0
- trail/stdlib/math.trail +41 -0
- trail/stdlib/stats.trail +17 -0
- trail/stdlib/timeseries.trail +20 -0
- trail/stdlib/transform.trail +10 -0
- trail/testing.py +91 -0
- trail/validate.py +140 -0
- trail_lang-0.1.0.dist-info/METADATA +101 -0
- trail_lang-0.1.0.dist-info/RECORD +35 -0
- trail_lang-0.1.0.dist-info/WHEEL +4 -0
- trail_lang-0.1.0.dist-info/entry_points.txt +5 -0
- trail_lang-0.1.0.dist-info/licenses/LICENSE +21 -0
trail/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
trail/ast.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Typed AST for Trail. All nodes are frozen dataclasses compared by value."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Literal:
|
|
9
|
+
value: float | int | str | bool
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class NameRef:
|
|
14
|
+
name: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class FieldRef:
|
|
19
|
+
path: tuple[str, ...]
|
|
20
|
+
source: str | None = None
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def column(self) -> str:
|
|
24
|
+
return ".".join(self.path)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class BinOp:
|
|
29
|
+
op: str # add sub mul div mod pow
|
|
30
|
+
left: "Expr"
|
|
31
|
+
right: "Expr"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Compare:
|
|
36
|
+
op: str # eq ne gt lt ge le
|
|
37
|
+
left: "Expr"
|
|
38
|
+
right: "Expr"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class In:
|
|
43
|
+
item: "Expr"
|
|
44
|
+
options: tuple[Literal, ...]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class BoolOp:
|
|
49
|
+
op: str # and or
|
|
50
|
+
left: "Expr"
|
|
51
|
+
right: "Expr"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Not:
|
|
56
|
+
operand: "Expr"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class Neg:
|
|
61
|
+
operand: "Expr"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Coalesce:
|
|
66
|
+
left: "Expr"
|
|
67
|
+
right: "Expr"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class Ternary:
|
|
72
|
+
value: "Expr"
|
|
73
|
+
cond: "Expr"
|
|
74
|
+
orelse: "Expr"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class Call:
|
|
79
|
+
name: str
|
|
80
|
+
args: tuple["Expr", ...]
|
|
81
|
+
kwargs: tuple[tuple[str, "Expr"], ...] = field(default=())
|
|
82
|
+
by: tuple[str, ...] | None = None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# --- declaration nodes (Task 3) ---
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True)
|
|
89
|
+
class Assignment:
|
|
90
|
+
name: str
|
|
91
|
+
expr: "Expr"
|
|
92
|
+
export: bool = False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True)
|
|
96
|
+
class ScoreCase:
|
|
97
|
+
value: "Expr"
|
|
98
|
+
cond: "Expr"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True)
|
|
102
|
+
class ScoreDecl:
|
|
103
|
+
name: str
|
|
104
|
+
weight: float
|
|
105
|
+
cases: tuple[ScoreCase, ...]
|
|
106
|
+
default: "Expr"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(frozen=True)
|
|
110
|
+
class UniverseDecl:
|
|
111
|
+
name: str
|
|
112
|
+
root: tuple[str, ...]
|
|
113
|
+
where: "Expr | None" = None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(frozen=True)
|
|
117
|
+
class ModelDecl:
|
|
118
|
+
name: str
|
|
119
|
+
universe: str | None
|
|
120
|
+
period: str
|
|
121
|
+
desc: str | None
|
|
122
|
+
on_missing: str
|
|
123
|
+
statements: tuple["Assignment | ScoreDecl", ...]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass(frozen=True)
|
|
127
|
+
class SignalDecl:
|
|
128
|
+
name: str
|
|
129
|
+
universe: str | None
|
|
130
|
+
period: str
|
|
131
|
+
expr: "Expr"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True)
|
|
135
|
+
class OpaqueDecl:
|
|
136
|
+
kind: str # strategy | backtest | learn | import
|
|
137
|
+
name: str
|
|
138
|
+
text: str = ""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@dataclass(frozen=True)
|
|
142
|
+
class FuncDef:
|
|
143
|
+
"""A user/stdlib function: a non-recursive expression macro over its parameters."""
|
|
144
|
+
|
|
145
|
+
name: str
|
|
146
|
+
params: tuple[str, ...]
|
|
147
|
+
body: "Expr"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True)
|
|
151
|
+
class Program:
|
|
152
|
+
decls: tuple[object, ...]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# --- REPL-dialect meta-commands (never valid in a model file; see reference §2.1) ---
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@dataclass(frozen=True)
|
|
159
|
+
class MetaCatalog:
|
|
160
|
+
"""`?` - the full catalog summary."""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@dataclass(frozen=True)
|
|
164
|
+
class MetaDescribe:
|
|
165
|
+
"""`?<target>` - describe a namespace, field, function, or source
|
|
166
|
+
(targets `functions` / `sources` list those categories)."""
|
|
167
|
+
|
|
168
|
+
target: tuple[str, ...]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
Expr = (
|
|
172
|
+
Literal | NameRef | FieldRef | BinOp | Compare | In | BoolOp | Not | Neg | Coalesce | Ternary | Call
|
|
173
|
+
)
|
trail/catalog.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Discovery core: catalog / describe over the schema, function, and source registries.
|
|
2
|
+
|
|
3
|
+
Shared engine behind every discovery front-end (REPL `?` meta-commands, the `trail
|
|
4
|
+
catalog` CLI, and - later - MCP tools and Jupyter magics). Returns CatalogResult (a
|
|
5
|
+
titled metadata table), never a panel: discovery is metadata, not computation.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from functools import lru_cache
|
|
11
|
+
|
|
12
|
+
import polars as pl
|
|
13
|
+
|
|
14
|
+
from trail import ast
|
|
15
|
+
from trail.config import DEFAULT_CONFIG, Config
|
|
16
|
+
from trail.schema import SCHEMA
|
|
17
|
+
from trail.validate import KNOWN_FUNCTIONS
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@lru_cache(maxsize=1)
|
|
21
|
+
def _stdlib_functions() -> dict[str, int]:
|
|
22
|
+
"""Bundled stdlib macro name -> parameter count (the derived layer). Internal
|
|
23
|
+
helpers (names starting with '_') are hidden from discovery."""
|
|
24
|
+
from trail.library import stdlib_source
|
|
25
|
+
from trail.macro import collect_functions
|
|
26
|
+
from trail.parser import parse_program
|
|
27
|
+
|
|
28
|
+
funcs = collect_functions(parse_program(stdlib_source()))
|
|
29
|
+
return {name: len(fd.params) for name, fd in funcs.items() if not name.startswith("_")}
|
|
30
|
+
|
|
31
|
+
# function -> (axis, one-line summary). Axis mirrors reference §7.
|
|
32
|
+
_FUNC_META: dict[str, tuple[str, str]] = {
|
|
33
|
+
"lag": ("time-series", "value n periods earlier (per security)"),
|
|
34
|
+
"roll_mean": ("time-series", "rolling mean over n periods"),
|
|
35
|
+
"roll_sum": ("time-series", "rolling sum over n periods"),
|
|
36
|
+
"roll_std": ("time-series", "rolling sample std (ddof=1) over n periods"),
|
|
37
|
+
"roll_var": ("time-series", "rolling sample variance over n periods"),
|
|
38
|
+
"roll_max": ("time-series", "rolling max over n periods"),
|
|
39
|
+
"roll_min": ("time-series", "rolling min over n periods"),
|
|
40
|
+
"roll_quantile": ("time-series", "rolling q-quantile (historical VaR)"),
|
|
41
|
+
"cummax": ("time-series", "expanding maximum"),
|
|
42
|
+
"cumsum": ("time-series", "expanding sum (discrete integral)"),
|
|
43
|
+
"cumprod": ("time-series", "expanding product (compounding)"),
|
|
44
|
+
"cummin": ("time-series", "expanding minimum"),
|
|
45
|
+
"roll_median": ("time-series", "rolling median over n periods"),
|
|
46
|
+
"roll_skew": ("time-series", "rolling skewness over n periods"),
|
|
47
|
+
"ewm_mean": ("time-series", "exponentially-weighted mean (span)"),
|
|
48
|
+
"ewm_std": ("time-series", "exponentially-weighted std (span)"),
|
|
49
|
+
"decay_linear": ("time-series", "linearly-decayed weighted mean over n periods"),
|
|
50
|
+
"zscore": ("cross-sectional", "standardize within (period[, group])"),
|
|
51
|
+
"rank": ("cross-sectional", "average-tie rank, ascending, within group"),
|
|
52
|
+
"winsorize": ("cross-sectional", "clip to [p, 1-p] group quantiles"),
|
|
53
|
+
"xs_mean": ("cross-sectional", "group mean, broadcast back to members"),
|
|
54
|
+
"xs_median": ("cross-sectional", "group median, broadcast back"),
|
|
55
|
+
"xs_sum": ("cross-sectional", "group sum, broadcast back"),
|
|
56
|
+
"xs_frac": ("cross-sectional", "fraction of group where cond is true"),
|
|
57
|
+
"xs_std": ("cross-sectional", "group sample std (ddof=1)"),
|
|
58
|
+
"xs_var": ("cross-sectional", "group sample variance"),
|
|
59
|
+
"xs_min": ("cross-sectional", "group minimum, broadcast back"),
|
|
60
|
+
"xs_max": ("cross-sectional", "group maximum, broadcast back"),
|
|
61
|
+
"xs_count": ("cross-sectional", "non-null count in group"),
|
|
62
|
+
"xs_quantile": ("cross-sectional", "group q-quantile, broadcast back"),
|
|
63
|
+
"count": ("elementwise", "sum of boolean flags as integers"),
|
|
64
|
+
"sqrt": ("elementwise", "square root (null for x<0)"),
|
|
65
|
+
"abs": ("elementwise", "absolute value"),
|
|
66
|
+
"log": ("elementwise", "natural log (null for x<=0)"),
|
|
67
|
+
"exp": ("elementwise", "e ** x"),
|
|
68
|
+
"sin": ("elementwise", "sine (radians)"),
|
|
69
|
+
"cos": ("elementwise", "cosine (radians)"),
|
|
70
|
+
"tan": ("elementwise", "tangent (radians)"),
|
|
71
|
+
"asin": ("elementwise", "arcsine"),
|
|
72
|
+
"acos": ("elementwise", "arccosine"),
|
|
73
|
+
"atan": ("elementwise", "arctangent"),
|
|
74
|
+
"floor": ("elementwise", "round down to integer"),
|
|
75
|
+
"ceil": ("elementwise", "round up to integer"),
|
|
76
|
+
"round": ("elementwise", "round to nearest integer"),
|
|
77
|
+
"clamp": ("elementwise", "clip x to [lo, hi]"),
|
|
78
|
+
"min": ("elementwise", "cell-wise min of two panels"),
|
|
79
|
+
"max": ("elementwise", "cell-wise max of two panels"),
|
|
80
|
+
"weighted_score": ("model", "weighted rollup of the model's score blocks"),
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class CatalogResult:
|
|
86
|
+
"""A titled metadata table. Renders in a terminal (str) and in Jupyter (_repr_html_)."""
|
|
87
|
+
|
|
88
|
+
title: str
|
|
89
|
+
frame: pl.DataFrame
|
|
90
|
+
|
|
91
|
+
def __str__(self) -> str:
|
|
92
|
+
return f"{self.title}\n{self.frame}"
|
|
93
|
+
|
|
94
|
+
def _repr_html_(self) -> str: # Jupyter rich display
|
|
95
|
+
return f"<strong>{self.title}</strong>{self.frame._repr_html_()}"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def namespaces() -> list[str]:
|
|
99
|
+
return sorted({c.split(".", 1)[0] for c in SCHEMA})
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def fields(namespace: str | None = None) -> CatalogResult:
|
|
103
|
+
items = [
|
|
104
|
+
(c, spec.kind) for c, spec in SCHEMA.items()
|
|
105
|
+
if namespace is None or c.split(".", 1)[0] == namespace
|
|
106
|
+
]
|
|
107
|
+
frame = pl.DataFrame({"field": [c for c, _ in items], "kind": [k for _, k in items]}).sort("field")
|
|
108
|
+
title = f"Fields in '{namespace}' ({frame.height})" if namespace else f"All fields ({frame.height})"
|
|
109
|
+
return CatalogResult(title, frame)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def functions() -> CatalogResult:
|
|
113
|
+
rows = {"function": [], "layer": [], "axis": [], "args": [], "summary": []}
|
|
114
|
+
for n in sorted(KNOWN_FUNCTIONS):
|
|
115
|
+
lo, hi = KNOWN_FUNCTIONS[n]
|
|
116
|
+
axis, summary = _FUNC_META.get(n, ("", ""))
|
|
117
|
+
rows["function"].append(n)
|
|
118
|
+
rows["layer"].append("primitive")
|
|
119
|
+
rows["axis"].append(axis)
|
|
120
|
+
rows["args"].append(str(lo) if lo == hi else f"{lo}..{hi}")
|
|
121
|
+
rows["summary"].append(summary)
|
|
122
|
+
std = _stdlib_functions()
|
|
123
|
+
for n in sorted(std):
|
|
124
|
+
rows["function"].append(n)
|
|
125
|
+
rows["layer"].append("derived")
|
|
126
|
+
rows["axis"].append("")
|
|
127
|
+
rows["args"].append(str(std[n]))
|
|
128
|
+
rows["summary"].append("stdlib macro")
|
|
129
|
+
return CatalogResult(f"Functions ({len(KNOWN_FUNCTIONS)} primitive + {len(std)} derived)",
|
|
130
|
+
pl.DataFrame(rows))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def sources(config: Config = DEFAULT_CONFIG) -> CatalogResult:
|
|
134
|
+
names = sorted(config.sources)
|
|
135
|
+
frame = pl.DataFrame({
|
|
136
|
+
"source": names,
|
|
137
|
+
"driver": [config.sources[n].driver for n in names],
|
|
138
|
+
})
|
|
139
|
+
precedence = ", ".join(f"{ns}=[{', '.join(chain)}]" for ns, chain in config.precedence.items())
|
|
140
|
+
return CatalogResult(f"Sources ({len(names)}) | precedence: {precedence}", frame)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _kv(title: str, pairs: list[tuple[str, str]]) -> CatalogResult:
|
|
144
|
+
frame = pl.DataFrame({"property": [k for k, _ in pairs], "value": [v for _, v in pairs]})
|
|
145
|
+
return CatalogResult(title, frame)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _source_detail(name: str, spec) -> CatalogResult:
|
|
149
|
+
"""Source metadata plus a best-effort coverage view for a discoverable source.
|
|
150
|
+
|
|
151
|
+
Instantiating a source can fail (e.g. a missing credential); that is reported
|
|
152
|
+
inline rather than raised, so discovery stays usable without a live connection.
|
|
153
|
+
"""
|
|
154
|
+
from trail.registry import resolve_driver
|
|
155
|
+
from trail.source import SupportsCapabilities, SupportsDiscovery
|
|
156
|
+
|
|
157
|
+
rows: list[tuple[str, str]] = [("driver", spec.driver), ("options", str(spec.options))]
|
|
158
|
+
try:
|
|
159
|
+
src = resolve_driver(spec.driver)(spec.options)
|
|
160
|
+
except Exception as e: # instantiation/credential failure: report, do not raise
|
|
161
|
+
rows.append(("discovery", f"unavailable ({e})"))
|
|
162
|
+
return _kv(f"Source {name}", rows)
|
|
163
|
+
try:
|
|
164
|
+
if isinstance(src, SupportsCapabilities):
|
|
165
|
+
caps = src.capabilities()
|
|
166
|
+
rows.append(("frequency", caps.frequency))
|
|
167
|
+
if caps.period_range:
|
|
168
|
+
rows.append(("period_range", f"{caps.period_range[0]}..{caps.period_range[1]}"))
|
|
169
|
+
if caps.provenance:
|
|
170
|
+
rows.append(("provenance", caps.provenance))
|
|
171
|
+
if isinstance(src, SupportsDiscovery):
|
|
172
|
+
all_fields = set(SCHEMA)
|
|
173
|
+
avail = src.available_fields()
|
|
174
|
+
rows.append(("provides", f"{len(avail & all_fields)}/{len(all_fields)} schema fields"))
|
|
175
|
+
missing = sorted(all_fields - avail)
|
|
176
|
+
if missing:
|
|
177
|
+
rows.append(("unavailable_fields", ", ".join(missing)))
|
|
178
|
+
else:
|
|
179
|
+
rows.append(("discovery", "not supported (core-tier source)"))
|
|
180
|
+
finally:
|
|
181
|
+
try:
|
|
182
|
+
src.close()
|
|
183
|
+
except Exception:
|
|
184
|
+
pass
|
|
185
|
+
return _kv(f"Source {name}", rows)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def describe(target: tuple[str, ...], config: Config = DEFAULT_CONFIG) -> CatalogResult:
|
|
189
|
+
dotted = ".".join(target)
|
|
190
|
+
# category list-alls
|
|
191
|
+
if target == ("functions",):
|
|
192
|
+
return functions()
|
|
193
|
+
if target == ("sources",):
|
|
194
|
+
return sources(config)
|
|
195
|
+
if target == ("fields",):
|
|
196
|
+
return fields()
|
|
197
|
+
# a specific field (dotted path in the schema)
|
|
198
|
+
if dotted in SCHEMA:
|
|
199
|
+
return _kv(f"Field {dotted}", [("column", dotted), ("kind", SCHEMA[dotted].kind)])
|
|
200
|
+
# a namespace
|
|
201
|
+
if len(target) == 1 and target[0] in namespaces():
|
|
202
|
+
return fields(target[0])
|
|
203
|
+
# a primitive function
|
|
204
|
+
if len(target) == 1 and target[0] in KNOWN_FUNCTIONS:
|
|
205
|
+
lo, hi = KNOWN_FUNCTIONS[target[0]]
|
|
206
|
+
axis, summary = _FUNC_META.get(target[0], ("", ""))
|
|
207
|
+
return _kv(f"Function {target[0]}", [
|
|
208
|
+
("layer", "primitive"), ("axis", axis),
|
|
209
|
+
("args", str(lo) if lo == hi else f"{lo}..{hi}"), ("summary", summary),
|
|
210
|
+
])
|
|
211
|
+
# a derived (stdlib macro) function
|
|
212
|
+
if len(target) == 1 and target[0] in _stdlib_functions():
|
|
213
|
+
return _kv(f"Function {target[0]}", [
|
|
214
|
+
("layer", "derived"), ("args", str(_stdlib_functions()[target[0]])),
|
|
215
|
+
("kind", "stdlib macro"),
|
|
216
|
+
])
|
|
217
|
+
# a source
|
|
218
|
+
if len(target) == 1 and target[0] in config.sources:
|
|
219
|
+
return _source_detail(target[0], config.sources[target[0]])
|
|
220
|
+
return CatalogResult(
|
|
221
|
+
f"Unknown catalog target: '{dotted}'",
|
|
222
|
+
pl.DataFrame({"hint": ["try ? , ?fields, ?functions, ?sources, or ?<namespace>"]}),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def catalog(config: Config = DEFAULT_CONFIG) -> CatalogResult:
|
|
227
|
+
ns = namespaces()
|
|
228
|
+
frame = pl.DataFrame({
|
|
229
|
+
"namespace": ns,
|
|
230
|
+
"fields": [len(fields(n).frame) for n in ns],
|
|
231
|
+
})
|
|
232
|
+
title = (f"Trail catalog - {len(SCHEMA)} fields across {len(ns)} namespaces, "
|
|
233
|
+
f"{len(KNOWN_FUNCTIONS)} primitive + {len(_stdlib_functions())} derived functions, "
|
|
234
|
+
f"{len(config.sources)} source(s). Use ?<namespace>, ?functions, ?sources, ?<name> for detail.")
|
|
235
|
+
return CatalogResult(title, frame)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def evaluate_meta(node, config: Config = DEFAULT_CONFIG) -> CatalogResult:
|
|
239
|
+
"""Route a parsed meta-command AST node to the discovery core."""
|
|
240
|
+
match node:
|
|
241
|
+
case ast.MetaCatalog():
|
|
242
|
+
return catalog(config)
|
|
243
|
+
case ast.MetaDescribe():
|
|
244
|
+
return describe(node.target, config)
|
|
245
|
+
raise TypeError(f"not a meta-command: {type(node).__name__}")
|
trail/cli.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Trail command line: validate and run .trail files (fixture-backed in this phase)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from lark.exceptions import UnexpectedInput
|
|
9
|
+
|
|
10
|
+
from trail import ast, catalog as catalog_core
|
|
11
|
+
from trail.compiler import compile_model
|
|
12
|
+
from trail.config import ConfigError, load_config
|
|
13
|
+
from trail.deps import extract
|
|
14
|
+
from trail.macro import TrailFunctionError
|
|
15
|
+
from trail.pipeline import prepare
|
|
16
|
+
from trail.sources import PanelConformanceWarning, load_panel_for
|
|
17
|
+
from trail.validate import validate
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@click.group()
|
|
21
|
+
def main() -> None:
|
|
22
|
+
"""Trail - financial expression language."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _load_and_validate(path: str, with_stdlib: bool = True) -> ast.Program:
|
|
26
|
+
with open(path) as fh:
|
|
27
|
+
src = fh.read()
|
|
28
|
+
try:
|
|
29
|
+
program = prepare(src, stdlib=with_stdlib) # prepend stdlib, parse, inline defs
|
|
30
|
+
except UnexpectedInput as e:
|
|
31
|
+
tok = getattr(e, "token", None)
|
|
32
|
+
detail = f": unexpected {str(tok)!r}" if tok else ""
|
|
33
|
+
click.echo(f"ERROR SYNTAX at line {e.line}, column {e.column}{detail}")
|
|
34
|
+
sys.exit(1)
|
|
35
|
+
except TrailFunctionError as e:
|
|
36
|
+
click.echo(f"ERROR FUNC {e}")
|
|
37
|
+
sys.exit(1)
|
|
38
|
+
issues = validate(program)
|
|
39
|
+
for i in issues:
|
|
40
|
+
click.echo(f"{'ERROR' if i.severity == 'error' else 'WARN '} {i.code} {i.message}")
|
|
41
|
+
if any(i.severity == "error" for i in issues):
|
|
42
|
+
sys.exit(1)
|
|
43
|
+
return program
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@main.command("validate")
|
|
47
|
+
@click.argument("path", type=click.Path(exists=True))
|
|
48
|
+
@click.option("--no-stdlib", is_flag=True, help="Do not load the bundled standard library.")
|
|
49
|
+
def validate_cmd(path: str, no_stdlib: bool) -> None:
|
|
50
|
+
_load_and_validate(path, with_stdlib=not no_stdlib)
|
|
51
|
+
click.echo("OK")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@main.command("catalog")
|
|
55
|
+
@click.argument("target", required=False)
|
|
56
|
+
@click.option("--config", "config_path", default=None, type=click.Path(exists=True))
|
|
57
|
+
def catalog_cmd(target: str | None, config_path: str | None) -> None:
|
|
58
|
+
"""Discover fields, functions, and sources. TARGET is a namespace, field,
|
|
59
|
+
function, source, or one of: fields, functions, sources. Same discovery core
|
|
60
|
+
as the REPL `?` meta-command."""
|
|
61
|
+
try:
|
|
62
|
+
config = load_config(config_path)
|
|
63
|
+
except ConfigError as e:
|
|
64
|
+
click.echo(f"ERROR CONFIG {e}")
|
|
65
|
+
sys.exit(1)
|
|
66
|
+
if target is None:
|
|
67
|
+
result = catalog_core.catalog(config)
|
|
68
|
+
else:
|
|
69
|
+
result = catalog_core.describe(tuple(target.split(".")), config)
|
|
70
|
+
click.echo(str(result))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@main.command("run")
|
|
74
|
+
@click.argument("path", type=click.Path(exists=True))
|
|
75
|
+
@click.option("--model", "model_name", required=True)
|
|
76
|
+
@click.option("--config", "config_path", default=None, type=click.Path(exists=True))
|
|
77
|
+
@click.option("--no-stdlib", is_flag=True, help="Do not load the bundled standard library.")
|
|
78
|
+
@click.option("--out", "out_path", default=None, type=click.Path())
|
|
79
|
+
def run_cmd(path: str, model_name: str, config_path: str | None, no_stdlib: bool, out_path: str | None) -> None:
|
|
80
|
+
program = _load_and_validate(path, with_stdlib=not no_stdlib)
|
|
81
|
+
try:
|
|
82
|
+
config = load_config(config_path)
|
|
83
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
84
|
+
warnings.simplefilter("always", PanelConformanceWarning)
|
|
85
|
+
panel = load_panel_for(config, set(extract(program).fields))
|
|
86
|
+
for w in caught:
|
|
87
|
+
if issubclass(w.category, PanelConformanceWarning):
|
|
88
|
+
click.echo(f"WARN {w.message}")
|
|
89
|
+
except ConfigError as e:
|
|
90
|
+
click.echo(f"ERROR CONFIG {e}")
|
|
91
|
+
sys.exit(1)
|
|
92
|
+
universes = {d.name: d for d in program.decls if isinstance(d, ast.UniverseDecl)}
|
|
93
|
+
models = {d.name: d for d in program.decls if isinstance(d, ast.ModelDecl)}
|
|
94
|
+
if model_name not in models:
|
|
95
|
+
click.echo(f"ERROR no model named '{model_name}'")
|
|
96
|
+
sys.exit(1)
|
|
97
|
+
result = compile_model(models[model_name], universes).run(panel)
|
|
98
|
+
if out_path:
|
|
99
|
+
result.write_parquet(out_path)
|
|
100
|
+
click.echo(f"wrote {out_path}")
|
|
101
|
+
else:
|
|
102
|
+
click.echo(str(result))
|
trail/compiler.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Lower AST to Polars expressions and executable model plans."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import polars as pl
|
|
8
|
+
|
|
9
|
+
from trail import ast
|
|
10
|
+
from trail.ops import PERIOD, SEC, build, safe_div
|
|
11
|
+
|
|
12
|
+
_BIN = {
|
|
13
|
+
"add": lambda x, y: x + y, "sub": lambda x, y: x - y, "mul": lambda x, y: x * y,
|
|
14
|
+
"mod": lambda x, y: x % y, "pow": lambda x, y: x.pow(y),
|
|
15
|
+
}
|
|
16
|
+
_CMP = {
|
|
17
|
+
"eq": lambda x, y: x == y, "ne": lambda x, y: x != y, "gt": lambda x, y: x > y,
|
|
18
|
+
"lt": lambda x, y: x < y, "ge": lambda x, y: x >= y, "le": lambda x, y: x <= y,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def compile_expr(e: ast.Expr, defined: set[str]) -> pl.Expr:
|
|
23
|
+
match e:
|
|
24
|
+
case ast.Literal():
|
|
25
|
+
return pl.lit(e.value)
|
|
26
|
+
case ast.NameRef():
|
|
27
|
+
return pl.col(e.name)
|
|
28
|
+
case ast.FieldRef():
|
|
29
|
+
return pl.col(e.column)
|
|
30
|
+
case ast.BinOp() if e.op == "div":
|
|
31
|
+
return safe_div(compile_expr(e.left, defined), compile_expr(e.right, defined))
|
|
32
|
+
case ast.BinOp():
|
|
33
|
+
return _BIN[e.op](compile_expr(e.left, defined), compile_expr(e.right, defined))
|
|
34
|
+
case ast.Compare():
|
|
35
|
+
return _CMP[e.op](compile_expr(e.left, defined), compile_expr(e.right, defined))
|
|
36
|
+
case ast.In():
|
|
37
|
+
return compile_expr(e.item, defined).is_in([o.value for o in e.options])
|
|
38
|
+
case ast.BoolOp() if e.op == "and":
|
|
39
|
+
return compile_expr(e.left, defined) & compile_expr(e.right, defined)
|
|
40
|
+
case ast.BoolOp():
|
|
41
|
+
return compile_expr(e.left, defined) | compile_expr(e.right, defined)
|
|
42
|
+
case ast.Not():
|
|
43
|
+
return ~compile_expr(e.operand, defined)
|
|
44
|
+
case ast.Neg():
|
|
45
|
+
return -compile_expr(e.operand, defined)
|
|
46
|
+
case ast.Coalesce():
|
|
47
|
+
return pl.coalesce([compile_expr(e.left, defined), compile_expr(e.right, defined)])
|
|
48
|
+
case ast.Ternary():
|
|
49
|
+
return (pl.when(compile_expr(e.cond, defined))
|
|
50
|
+
.then(compile_expr(e.value, defined))
|
|
51
|
+
.otherwise(compile_expr(e.orelse, defined)))
|
|
52
|
+
case ast.Call():
|
|
53
|
+
args = [_call_arg(a, defined) for a in e.args]
|
|
54
|
+
kwargs = {k: _call_arg(v, defined) for k, v in e.kwargs}
|
|
55
|
+
return build(e.name, args, kwargs, e.by)
|
|
56
|
+
raise TypeError(f"cannot compile {type(e).__name__}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _call_arg(a: ast.Expr, defined: set[str]):
|
|
60
|
+
# numeric literals pass through as python numbers (window sizes, quantiles)
|
|
61
|
+
if isinstance(a, ast.Literal) and isinstance(a.value, (int, float)) and not isinstance(a.value, bool):
|
|
62
|
+
return a.value
|
|
63
|
+
return compile_expr(a, defined)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _score_expr(sd: ast.ScoreDecl, defined: set[str]) -> pl.Expr:
|
|
67
|
+
"""First-match-wins cases. Null iff every condition is null (all inputs missing),
|
|
68
|
+
which is what makes `on_missing skip` renormalization meaningful (reference §4.3/§7.5)."""
|
|
69
|
+
conds = [compile_expr(c.cond, defined) for c in sd.cases]
|
|
70
|
+
node = pl.when(conds[0]).then(compile_expr(sd.cases[0].value, defined))
|
|
71
|
+
for cond, case in zip(conds[1:], sd.cases[1:], strict=True):
|
|
72
|
+
node = node.when(cond).then(compile_expr(case.value, defined))
|
|
73
|
+
normal = node.otherwise(compile_expr(sd.default, defined)).cast(pl.Float64)
|
|
74
|
+
all_null = conds[0].is_null()
|
|
75
|
+
for cond in conds[1:]:
|
|
76
|
+
all_null = all_null & cond.is_null()
|
|
77
|
+
return pl.when(all_null).then(pl.lit(None, dtype=pl.Float64)).otherwise(normal)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _score_max(sd: ast.ScoreDecl) -> float:
|
|
81
|
+
return float(max([c.value.value for c in sd.cases] + [sd.default.value]))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _weighted_score(scores: list[ast.ScoreDecl], on_missing: str) -> pl.Expr:
|
|
85
|
+
num = pl.lit(0.0)
|
|
86
|
+
den = pl.lit(0.0)
|
|
87
|
+
for sd in scores:
|
|
88
|
+
s = pl.col(sd.name)
|
|
89
|
+
w, mx = sd.weight, _score_max(sd)
|
|
90
|
+
num = num + pl.coalesce([s * w, pl.lit(0.0)])
|
|
91
|
+
if on_missing == "zero":
|
|
92
|
+
den = den + pl.lit(w * mx)
|
|
93
|
+
else: # skip (median compiles like skip this phase)
|
|
94
|
+
den = den + pl.when(s.is_not_null()).then(w * mx).otherwise(0.0)
|
|
95
|
+
return safe_div(num, den)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _is_weighted_score(expr: ast.Expr) -> bool:
|
|
99
|
+
return isinstance(expr, ast.Call) and expr.name == "weighted_score"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class ModelPlan:
|
|
104
|
+
_lf_builder: Callable[[pl.DataFrame], pl.LazyFrame]
|
|
105
|
+
exports: tuple[str, ...]
|
|
106
|
+
|
|
107
|
+
def run(self, panel: pl.DataFrame) -> pl.DataFrame:
|
|
108
|
+
return self._lf_builder(panel).select([SEC, PERIOD, *self.exports]).collect()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def compile_model(model: ast.ModelDecl, universes: dict[str, ast.UniverseDecl]) -> ModelPlan:
|
|
112
|
+
# Universe binding per reference §8.3: explicit `on` wins; a sole declared
|
|
113
|
+
# universe auto-binds; zero universes = full panel.
|
|
114
|
+
if model.universe is not None:
|
|
115
|
+
uni = universes.get(model.universe)
|
|
116
|
+
elif len(universes) == 1:
|
|
117
|
+
uni = next(iter(universes.values()))
|
|
118
|
+
else:
|
|
119
|
+
uni = None
|
|
120
|
+
scores = [s for s in model.statements if isinstance(s, ast.ScoreDecl)]
|
|
121
|
+
|
|
122
|
+
def builder(panel: pl.DataFrame) -> pl.LazyFrame:
|
|
123
|
+
lf = panel.lazy()
|
|
124
|
+
if uni is not None and uni.where is not None:
|
|
125
|
+
lf = lf.filter(compile_expr(uni.where, set()))
|
|
126
|
+
defined: set[str] = set()
|
|
127
|
+
for st in model.statements:
|
|
128
|
+
if isinstance(st, ast.ScoreDecl):
|
|
129
|
+
lf = lf.with_columns(_score_expr(st, defined).alias(st.name))
|
|
130
|
+
defined.add(st.name)
|
|
131
|
+
elif _is_weighted_score(st.expr):
|
|
132
|
+
lf = lf.with_columns(_weighted_score(scores, model.on_missing).alias(st.name))
|
|
133
|
+
defined.add(st.name)
|
|
134
|
+
else:
|
|
135
|
+
lf = lf.with_columns(compile_expr(st.expr, defined).alias(st.name))
|
|
136
|
+
defined.add(st.name)
|
|
137
|
+
return lf
|
|
138
|
+
|
|
139
|
+
exports = tuple(s.name for s in model.statements if isinstance(s, ast.Assignment) and s.export)
|
|
140
|
+
return ModelPlan(builder, exports)
|