smartpipe-cli 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smartpipe/__init__.py +6 -0
- smartpipe/__main__.py +8 -0
- smartpipe/assets/probe.png +0 -0
- smartpipe/assets/probe.txt +1 -0
- smartpipe/assets/probe.wav +0 -0
- smartpipe/cli/__init__.py +5 -0
- smartpipe/cli/auth_cmd.py +78 -0
- smartpipe/cli/cache_cmd.py +60 -0
- smartpipe/cli/chart_cmd.py +60 -0
- smartpipe/cli/cite_cmd.py +26 -0
- smartpipe/cli/cluster_cmd.py +102 -0
- smartpipe/cli/completions.py +91 -0
- smartpipe/cli/config_cmd.py +234 -0
- smartpipe/cli/diff_cmd.py +100 -0
- smartpipe/cli/distinct_cmd.py +94 -0
- smartpipe/cli/doctor_cmd.py +207 -0
- smartpipe/cli/echo_cmd.py +44 -0
- smartpipe/cli/embed_cmd.py +80 -0
- smartpipe/cli/extend_cmd.py +138 -0
- smartpipe/cli/filter_cmd.py +87 -0
- smartpipe/cli/getschema_cmd.py +31 -0
- smartpipe/cli/input_options.py +113 -0
- smartpipe/cli/interrupts.py +92 -0
- smartpipe/cli/join_cmd.py +162 -0
- smartpipe/cli/map_cmd.py +150 -0
- smartpipe/cli/outliers_cmd.py +82 -0
- smartpipe/cli/probe_cmd.py +223 -0
- smartpipe/cli/reduce_cmd.py +129 -0
- smartpipe/cli/root.py +281 -0
- smartpipe/cli/run_cmd.py +136 -0
- smartpipe/cli/sample_cmd.py +35 -0
- smartpipe/cli/schema_cmd.py +75 -0
- smartpipe/cli/screens.py +231 -0
- smartpipe/cli/sem_file.py +453 -0
- smartpipe/cli/sort_cmd.py +33 -0
- smartpipe/cli/split_cmd.py +76 -0
- smartpipe/cli/summarize_cmd.py +37 -0
- smartpipe/cli/top_k_cmd.py +97 -0
- smartpipe/cli/usage_cmd.py +66 -0
- smartpipe/cli/where_cmd.py +36 -0
- smartpipe/config/__init__.py +5 -0
- smartpipe/config/credentials.py +118 -0
- smartpipe/config/display.py +70 -0
- smartpipe/config/doctor.py +58 -0
- smartpipe/config/paths.py +38 -0
- smartpipe/config/store.py +252 -0
- smartpipe/container.py +439 -0
- smartpipe/core/__init__.py +5 -0
- smartpipe/core/errors.py +57 -0
- smartpipe/core/jsontools.py +56 -0
- smartpipe/engine/__init__.py +9 -0
- smartpipe/engine/aggregate.py +234 -0
- smartpipe/engine/blocking.py +44 -0
- smartpipe/engine/chart.py +143 -0
- smartpipe/engine/chunking.py +161 -0
- smartpipe/engine/clustering.py +94 -0
- smartpipe/engine/predicate.py +330 -0
- smartpipe/engine/prompts.py +601 -0
- smartpipe/engine/ranking.py +62 -0
- smartpipe/engine/runner.py +175 -0
- smartpipe/engine/schema.py +208 -0
- smartpipe/engine/schema_dsl.py +144 -0
- smartpipe/engine/tally.py +53 -0
- smartpipe/engine/timebin.py +67 -0
- smartpipe/engine/units.py +43 -0
- smartpipe/engine/windows.py +78 -0
- smartpipe/io/__init__.py +9 -0
- smartpipe/io/diagnostics.py +148 -0
- smartpipe/io/inputs.py +44 -0
- smartpipe/io/items.py +149 -0
- smartpipe/io/leaderboard.py +52 -0
- smartpipe/io/metering.py +180 -0
- smartpipe/io/progress.py +140 -0
- smartpipe/io/readers.py +455 -0
- smartpipe/io/text.py +40 -0
- smartpipe/io/tty.py +88 -0
- smartpipe/io/usage.py +214 -0
- smartpipe/io/writers.py +340 -0
- smartpipe/models/__init__.py +5 -0
- smartpipe/models/anthropic_adapter.py +149 -0
- smartpipe/models/base.py +170 -0
- smartpipe/models/budget.py +94 -0
- smartpipe/models/cache.py +132 -0
- smartpipe/models/gemini_native.py +196 -0
- smartpipe/models/http_support.py +77 -0
- smartpipe/models/jina.py +98 -0
- smartpipe/models/local_embed.py +76 -0
- smartpipe/models/ollama.py +204 -0
- smartpipe/models/openai_codex.py +237 -0
- smartpipe/models/openai_compat.py +328 -0
- smartpipe/models/openai_oauth.py +366 -0
- smartpipe/models/resolve.py +78 -0
- smartpipe/models/retry.py +69 -0
- smartpipe/models/stt.py +80 -0
- smartpipe/models/windows.py +116 -0
- smartpipe/parsing/__init__.py +10 -0
- smartpipe/parsing/detect.py +178 -0
- smartpipe/parsing/extract.py +582 -0
- smartpipe/py.typed +0 -0
- smartpipe/verbs/__init__.py +5 -0
- smartpipe/verbs/chart.py +153 -0
- smartpipe/verbs/cluster.py +220 -0
- smartpipe/verbs/common.py +468 -0
- smartpipe/verbs/convert.py +251 -0
- smartpipe/verbs/diff.py +206 -0
- smartpipe/verbs/distinct.py +164 -0
- smartpipe/verbs/embed.py +166 -0
- smartpipe/verbs/extend.py +180 -0
- smartpipe/verbs/filter.py +191 -0
- smartpipe/verbs/getschema.py +135 -0
- smartpipe/verbs/join.py +413 -0
- smartpipe/verbs/map.py +315 -0
- smartpipe/verbs/outliers.py +119 -0
- smartpipe/verbs/reduce.py +428 -0
- smartpipe/verbs/sample.py +52 -0
- smartpipe/verbs/sortverb.py +63 -0
- smartpipe/verbs/split.py +333 -0
- smartpipe/verbs/summarize.py +60 -0
- smartpipe/verbs/top_k.py +318 -0
- smartpipe/verbs/where.py +47 -0
- smartpipe_cli-1.3.0.dist-info/METADATA +192 -0
- smartpipe_cli-1.3.0.dist-info/RECORD +126 -0
- smartpipe_cli-1.3.0.dist-info/WHEEL +4 -0
- smartpipe_cli-1.3.0.dist-info/entry_points.txt +3 -0
- smartpipe_cli-1.3.0.dist-info/licenses/LICENSE +201 -0
- smartpipe_cli-1.3.0.dist-info/licenses/NOTICE +5 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Ordered, bounded-concurrency execution — the one primitive every per-item
|
|
2
|
+
verb runs on (plan/architecture.md "Execution engine").
|
|
3
|
+
|
|
4
|
+
Guarantees (all property-tested):
|
|
5
|
+
1. Order — outcomes yield in input order, always. Grep-shaped tools that reorder
|
|
6
|
+
lines break every downstream diff/paste/log habit.
|
|
7
|
+
2. Boundedness — at most ``concurrency`` workers in flight; memory O(concurrency)
|
|
8
|
+
regardless of input size, so it streams unbounded input the same as a batch.
|
|
9
|
+
3. Isolation — a worker raising ``ItemError`` yields ``Skipped`` and the run
|
|
10
|
+
continues; any other exception propagates (it's a bug, crash loudly).
|
|
11
|
+
4. Accounting — once enough items finish, a majority-failure run halts with
|
|
12
|
+
``TooManyFailures`` rather than burning the whole input on a broken config.
|
|
13
|
+
|
|
14
|
+
The worker is injected (a first-class async function), so this module does no
|
|
15
|
+
I/O of its own and orders purely by arrival.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import TYPE_CHECKING, Generic, TypeVar
|
|
23
|
+
|
|
24
|
+
from smartpipe.core.errors import ItemError, TooManyFailures
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
28
|
+
|
|
29
|
+
from smartpipe.io.items import Item, ItemSource
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"Done",
|
|
33
|
+
"FailurePolicy",
|
|
34
|
+
"ItemOutcome",
|
|
35
|
+
"Skipped",
|
|
36
|
+
"run_ordered",
|
|
37
|
+
"should_halt",
|
|
38
|
+
"should_halt_consecutive",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
R = TypeVar("R")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class Done(Generic[R]):
|
|
46
|
+
index: int
|
|
47
|
+
value: R
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True, slots=True)
|
|
51
|
+
class Skipped:
|
|
52
|
+
index: int
|
|
53
|
+
reason: str
|
|
54
|
+
source: ItemSource
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
ItemOutcome = Done[R] | Skipped
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True, slots=True)
|
|
61
|
+
class FailurePolicy:
|
|
62
|
+
halt_ratio: float = 0.5
|
|
63
|
+
min_sample: int = 20
|
|
64
|
+
consecutive_limit: int = 5 # D18: a doomed run must not wait for the ratio
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def should_halt(policy: FailurePolicy, *, total: int, skipped: int) -> bool:
|
|
68
|
+
"""True once enough items have finished and a majority of them failed.
|
|
69
|
+
``min_sample`` prevents a 3-item pipe halting on 2 flukes."""
|
|
70
|
+
return total >= policy.min_sample and skipped > total * policy.halt_ratio
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def should_halt_consecutive(policy: FailurePolicy, *, succeeded: bool, consecutive: int) -> bool:
|
|
74
|
+
"""D18's cost guardrail: N consecutive failures with zero successes *ever* means
|
|
75
|
+
the run was doomed from item 1 — stop paying. Any success disarms this rule
|
|
76
|
+
permanently (a working run with a bad patch of input must not die early)."""
|
|
77
|
+
return not succeeded and consecutive >= policy.consecutive_limit
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def run_ordered(
|
|
81
|
+
items: AsyncIterator[Item],
|
|
82
|
+
worker: Callable[[Item], Awaitable[R]],
|
|
83
|
+
*,
|
|
84
|
+
concurrency: int,
|
|
85
|
+
failure_policy: FailurePolicy,
|
|
86
|
+
stop: asyncio.Event | None = None,
|
|
87
|
+
) -> AsyncIterator[ItemOutcome[R]]:
|
|
88
|
+
"""``stop`` (set by the interrupt shell) halts *intake*: no new workers spawn,
|
|
89
|
+
but everything already in flight completes and is emitted in order — the drain
|
|
90
|
+
contract of ux.md §12.
|
|
91
|
+
|
|
92
|
+
Intake runs as its OWN task so that waiting for the next input item never
|
|
93
|
+
blocks the emission of already-completed outcomes — the streaming property
|
|
94
|
+
(a live stream can pause mid-flow; results must still come out).
|
|
95
|
+
"""
|
|
96
|
+
item_iter = aiter(items)
|
|
97
|
+
pending: dict[int, asyncio.Task[ItemOutcome[R]]] = {}
|
|
98
|
+
slots = asyncio.Semaphore(concurrency)
|
|
99
|
+
progressed = asyncio.Event() # set whenever intake adds a task or finishes
|
|
100
|
+
intake_done = False
|
|
101
|
+
next_to_emit = 0
|
|
102
|
+
total = 0
|
|
103
|
+
skipped = 0
|
|
104
|
+
consecutive = 0
|
|
105
|
+
succeeded = False
|
|
106
|
+
|
|
107
|
+
def stopping() -> bool:
|
|
108
|
+
return stop is not None and stop.is_set()
|
|
109
|
+
|
|
110
|
+
async def intake() -> None:
|
|
111
|
+
nonlocal intake_done
|
|
112
|
+
index = 0
|
|
113
|
+
try:
|
|
114
|
+
while not stopping():
|
|
115
|
+
await slots.acquire()
|
|
116
|
+
if stopping(): # woke up into a drain — don't start new work
|
|
117
|
+
slots.release()
|
|
118
|
+
break
|
|
119
|
+
try:
|
|
120
|
+
item = await anext(item_iter)
|
|
121
|
+
except StopAsyncIteration:
|
|
122
|
+
slots.release()
|
|
123
|
+
break
|
|
124
|
+
pending[index] = asyncio.create_task(_run_one(worker, item))
|
|
125
|
+
index += 1
|
|
126
|
+
progressed.set()
|
|
127
|
+
finally:
|
|
128
|
+
intake_done = True
|
|
129
|
+
progressed.set()
|
|
130
|
+
|
|
131
|
+
intake_task = asyncio.create_task(intake())
|
|
132
|
+
try:
|
|
133
|
+
while True:
|
|
134
|
+
task = pending.get(next_to_emit)
|
|
135
|
+
if task is None:
|
|
136
|
+
if intake_done and not pending:
|
|
137
|
+
return
|
|
138
|
+
progressed.clear()
|
|
139
|
+
# re-check before sleeping: intake may have progressed between the
|
|
140
|
+
# get() above and the clear() — a real race, so the branch can't be
|
|
141
|
+
# hit deterministically in a test; excluded rather than pretended at.
|
|
142
|
+
if next_to_emit in pending or (intake_done and not pending): # pragma: no cover
|
|
143
|
+
continue # pragma: no cover
|
|
144
|
+
await progressed.wait()
|
|
145
|
+
continue
|
|
146
|
+
outcome = await task
|
|
147
|
+
del pending[next_to_emit]
|
|
148
|
+
slots.release()
|
|
149
|
+
next_to_emit += 1
|
|
150
|
+
yield outcome
|
|
151
|
+
total += 1
|
|
152
|
+
if isinstance(outcome, Skipped):
|
|
153
|
+
skipped += 1
|
|
154
|
+
consecutive += 1
|
|
155
|
+
if should_halt(failure_policy, total=total, skipped=skipped):
|
|
156
|
+
raise TooManyFailures(skipped, total, outcome.reason)
|
|
157
|
+
if should_halt_consecutive(
|
|
158
|
+
failure_policy, succeeded=succeeded, consecutive=consecutive
|
|
159
|
+
):
|
|
160
|
+
raise TooManyFailures(skipped, total, outcome.reason)
|
|
161
|
+
else:
|
|
162
|
+
consecutive = 0
|
|
163
|
+
succeeded = True
|
|
164
|
+
finally:
|
|
165
|
+
intake_task.cancel()
|
|
166
|
+
for task in pending.values():
|
|
167
|
+
task.cancel()
|
|
168
|
+
await asyncio.gather(intake_task, *pending.values(), return_exceptions=True)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
async def _run_one(worker: Callable[[Item], Awaitable[R]], item: Item) -> ItemOutcome[R]:
|
|
172
|
+
try:
|
|
173
|
+
return Done(item.source.index, await worker(item))
|
|
174
|
+
except ItemError as exc:
|
|
175
|
+
return Skipped(item.source.index, str(exc), item.source)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Structured-output schemas (plan/decisions.md D07): shorthand synthesis,
|
|
2
|
+
JSON-Schema loading, and validate-with-light-coercion of a model reply.
|
|
3
|
+
|
|
4
|
+
``validate_and_coerce`` raises ``ItemError`` on any failure, with a message that
|
|
5
|
+
names the problem — the ``map`` verb feeds that message back to the model as the
|
|
6
|
+
single repair retry, so the message is repair context, not just a log line.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
from smartpipe.core.errors import ItemError, SetupFault
|
|
16
|
+
from smartpipe.core.jsontools import as_items, as_record, as_str
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from collections.abc import Mapping, Sequence
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"is_strict_compatible",
|
|
24
|
+
"load_schema",
|
|
25
|
+
"parse_schema_draft",
|
|
26
|
+
"shorthand_to_schema",
|
|
27
|
+
"validate_and_coerce",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
_FENCE = re.compile(r"^```[A-Za-z0-9]*\n?|\n?```$")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def shorthand_to_schema(
|
|
34
|
+
fields: Sequence[str],
|
|
35
|
+
*,
|
|
36
|
+
descriptions: Mapping[str, str] | None = None,
|
|
37
|
+
types: Mapping[str, Mapping[str, object]] | None = None,
|
|
38
|
+
) -> dict[str, object]:
|
|
39
|
+
"""Turn ``{vendor, total}`` fields into a strict JSON Schema. Inline types
|
|
40
|
+
(D37) and rung-2 descriptions (D22) ride each property; a fully-typed group
|
|
41
|
+
regains strict mode (every property carries a type)."""
|
|
42
|
+
notes = descriptions or {}
|
|
43
|
+
typed = types or {}
|
|
44
|
+
|
|
45
|
+
def _property(field: str) -> dict[str, object]:
|
|
46
|
+
prop: dict[str, object] = dict(typed.get(field, {}))
|
|
47
|
+
if field in notes:
|
|
48
|
+
prop["description"] = notes[field]
|
|
49
|
+
return prop
|
|
50
|
+
|
|
51
|
+
properties: dict[str, object] = {field: _property(field) for field in fields}
|
|
52
|
+
return {
|
|
53
|
+
"type": "object",
|
|
54
|
+
"properties": properties,
|
|
55
|
+
"required": list(fields),
|
|
56
|
+
"additionalProperties": False,
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def is_strict_compatible(schema: Mapping[str, object]) -> bool:
|
|
61
|
+
"""Would OpenAI/Mistral ``strict: true`` json_schema mode accept this schema?
|
|
62
|
+
|
|
63
|
+
Strict mode demands, at every object layer: every property listed in
|
|
64
|
+
``required`` and ``additionalProperties: false``. Brace-shorthand schemas
|
|
65
|
+
qualify by construction; a user ``--schema`` with optional fields does not —
|
|
66
|
+
claiming strict for it draws a 400 and skips items for the wrong reason.
|
|
67
|
+
"""
|
|
68
|
+
if _looks_like_object(schema):
|
|
69
|
+
if schema.get("additionalProperties") is not False:
|
|
70
|
+
return False
|
|
71
|
+
properties = as_record(schema.get("properties")) or {}
|
|
72
|
+
required = as_items(schema.get("required")) or ()
|
|
73
|
+
required_names = {name for name in required if isinstance(name, str)}
|
|
74
|
+
if not set(properties) <= required_names:
|
|
75
|
+
return False
|
|
76
|
+
for value in properties.values():
|
|
77
|
+
child = as_record(value)
|
|
78
|
+
# live-caught (2026-07-05): strict mode also demands a 'type' per
|
|
79
|
+
# property — an untyped {} (the brace shorthand) draws the same 400
|
|
80
|
+
if child is None or ("type" not in child and "enum" not in child):
|
|
81
|
+
return False
|
|
82
|
+
if not is_strict_compatible(child):
|
|
83
|
+
return False
|
|
84
|
+
items = as_record(schema.get("items"))
|
|
85
|
+
return items is None or is_strict_compatible(items)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _looks_like_object(schema: Mapping[str, object]) -> bool:
|
|
89
|
+
return schema.get("type") == "object" or as_record(schema.get("properties")) is not None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def load_schema(path: Path) -> dict[str, object]:
|
|
93
|
+
if not path.exists():
|
|
94
|
+
raise SetupFault(
|
|
95
|
+
f"error: no schema file at {path}\n Check the --schema path and try again."
|
|
96
|
+
)
|
|
97
|
+
try:
|
|
98
|
+
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
99
|
+
except json.JSONDecodeError as exc:
|
|
100
|
+
raise SetupFault(
|
|
101
|
+
f"error: {path} isn't valid JSON\n {exc}\n A --schema file must be a JSON Schema."
|
|
102
|
+
) from exc
|
|
103
|
+
record = as_record(loaded)
|
|
104
|
+
if record is None:
|
|
105
|
+
raise SetupFault(
|
|
106
|
+
f"error: {path} isn't a JSON Schema\n The top level must be a JSON object."
|
|
107
|
+
)
|
|
108
|
+
return dict(record)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def parse_schema_draft(reply: str) -> dict[str, object]:
|
|
112
|
+
"""Rung 4 (D22): a model-drafted schema, validated against the JSON-Schema
|
|
113
|
+
meta-schema AND our loader rules — ``ItemError`` names what's wrong (repair
|
|
114
|
+
context). An invalid draft must never reach stdout."""
|
|
115
|
+
import jsonschema
|
|
116
|
+
|
|
117
|
+
record = as_record(_extract_json(reply)) # ItemError when there's no JSON at all
|
|
118
|
+
if record is None:
|
|
119
|
+
raise ItemError("the draft isn't a JSON object")
|
|
120
|
+
candidate = dict(record)
|
|
121
|
+
try:
|
|
122
|
+
jsonschema.Draft202012Validator.check_schema(candidate)
|
|
123
|
+
except jsonschema.SchemaError as exc:
|
|
124
|
+
raise ItemError(f"not a valid JSON Schema: {exc.message}") from exc
|
|
125
|
+
return candidate
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def validate_and_coerce(reply: str, schema: Mapping[str, object]) -> dict[str, object]:
|
|
129
|
+
import jsonschema # function-local: --help must not pay for the validator stack
|
|
130
|
+
|
|
131
|
+
record = as_record(_extract_json(reply))
|
|
132
|
+
if record is None:
|
|
133
|
+
raise ItemError("model returned JSON but not an object")
|
|
134
|
+
coerced = _coerce(record, schema)
|
|
135
|
+
trimmed = _drop_extra(coerced, schema)
|
|
136
|
+
try:
|
|
137
|
+
jsonschema.validate(trimmed, dict(schema))
|
|
138
|
+
except jsonschema.ValidationError as exc:
|
|
139
|
+
raise ItemError(f"output does not match the schema: {exc.message}") from exc
|
|
140
|
+
return trimmed
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _extract_json(reply: str) -> object:
|
|
144
|
+
text = reply.strip()
|
|
145
|
+
if text.startswith("```"):
|
|
146
|
+
text = _FENCE.sub("", text).strip()
|
|
147
|
+
for candidate in _json_candidates(text):
|
|
148
|
+
try:
|
|
149
|
+
return json.loads(candidate)
|
|
150
|
+
except json.JSONDecodeError:
|
|
151
|
+
continue
|
|
152
|
+
raise ItemError("model did not return valid JSON")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _json_candidates(text: str) -> tuple[str, ...]:
|
|
156
|
+
start = text.find("{")
|
|
157
|
+
end = text.rfind("}")
|
|
158
|
+
if start != -1 and end > start:
|
|
159
|
+
return (text, text[start : end + 1])
|
|
160
|
+
return (text,)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _coerce(record: Mapping[str, object], schema: Mapping[str, object]) -> dict[str, object]:
|
|
164
|
+
properties = as_record(schema.get("properties")) or {}
|
|
165
|
+
return {
|
|
166
|
+
key: _coerce_scalar(value, as_record(properties.get(key))) for key, value in record.items()
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _coerce_scalar(value: object, prop_schema: Mapping[str, object] | None) -> object:
|
|
171
|
+
if prop_schema is None or not isinstance(value, str):
|
|
172
|
+
return value
|
|
173
|
+
text = value.strip()
|
|
174
|
+
match as_str(prop_schema.get("type")):
|
|
175
|
+
case "integer":
|
|
176
|
+
return int(text) if _is_int(text) else value
|
|
177
|
+
case "number":
|
|
178
|
+
return float(text) if _is_float(text) else value
|
|
179
|
+
case "boolean":
|
|
180
|
+
return _as_bool(text, fallback=value)
|
|
181
|
+
case "null":
|
|
182
|
+
return None if text == "null" else value
|
|
183
|
+
case _:
|
|
184
|
+
return value
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _drop_extra(record: dict[str, object], schema: Mapping[str, object]) -> dict[str, object]:
|
|
188
|
+
if schema.get("additionalProperties") is not False:
|
|
189
|
+
return record
|
|
190
|
+
allowed = as_record(schema.get("properties")) or {}
|
|
191
|
+
return {key: value for key, value in record.items() if key in allowed}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _is_int(text: str) -> bool:
|
|
195
|
+
return bool(re.fullmatch(r"[+-]?\d+", text))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _is_float(text: str) -> bool:
|
|
199
|
+
return bool(re.fullmatch(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?", text))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _as_bool(text: str, *, fallback: object) -> object:
|
|
203
|
+
lowered = text.lower()
|
|
204
|
+
if lowered == "true":
|
|
205
|
+
return True
|
|
206
|
+
if lowered == "false":
|
|
207
|
+
return False
|
|
208
|
+
return fallback
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Rung 3 of the schema ladder (D22): a deterministic mini-DSL → JSON Schema.
|
|
2
|
+
|
|
3
|
+
``vendor string; total number >= 0; status enum(paid, unpaid)`` — parsed with
|
|
4
|
+
zero model calls and zero I/O, so a typo fails at argv time, before anything
|
|
5
|
+
could cost money (D18 applied to schemas). Everything richer than this grammar
|
|
6
|
+
belongs in a ``--schema`` file; the error screens say so.
|
|
7
|
+
|
|
8
|
+
Grammar (pinned in ux.md):
|
|
9
|
+
fields ::= field (";" field)*
|
|
10
|
+
field ::= name type constraint*
|
|
11
|
+
type ::= string | number | integer | boolean | enum(a, b, …)
|
|
12
|
+
| string[] | number[]
|
|
13
|
+
constraint ::= ">=" N | "<=" N | minLength=N | maxLength=N | optional
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
from smartpipe.core.errors import UsageFault
|
|
21
|
+
|
|
22
|
+
__all__ = ["TYPE_MENU", "dsl_to_schema", "type_token"]
|
|
23
|
+
|
|
24
|
+
_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
|
25
|
+
_HELP = (
|
|
26
|
+
"\n Types: string · number · integer · boolean · enum(a, b, …) · string[] · number[]"
|
|
27
|
+
"\n Constraints: >= N · <= N · minLength=N · maxLength=N · optional"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_SIMPLE_TYPES: dict[str, dict[str, object]] = {
|
|
31
|
+
"string": {"type": "string"},
|
|
32
|
+
"number": {"type": "number"},
|
|
33
|
+
"integer": {"type": "integer"},
|
|
34
|
+
"boolean": {"type": "boolean"},
|
|
35
|
+
"string[]": {"type": "array", "items": {"type": "string"}},
|
|
36
|
+
"number[]": {"type": "array", "items": {"type": "number"}},
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_BOUND = re.compile(r"(>=|<=)\s*(-?\d+(?:\.\d+)?)")
|
|
40
|
+
_LENGTH = re.compile(r"(minLength|maxLength)=(\d+)")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
TYPE_MENU = "string · number · integer · boolean · enum(a, b, …) · string[] · number[]"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def type_token(token: str) -> dict[str, object] | None:
|
|
47
|
+
"""One type token → a property dict, or None when it isn't a type.
|
|
48
|
+
|
|
49
|
+
Shared vocabulary with the braces (D37): one grammar, two homes."""
|
|
50
|
+
simple = _SIMPLE_TYPES.get(token)
|
|
51
|
+
if simple is not None:
|
|
52
|
+
return dict(simple)
|
|
53
|
+
if token.startswith("enum(") and token.endswith(")"):
|
|
54
|
+
values = [value.strip() for value in token[5:-1].split(",") if value.strip()]
|
|
55
|
+
return {"enum": values} if values else None
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def dsl_to_schema(text: str) -> dict[str, object]:
|
|
60
|
+
"""Parse the ``--schema-from`` DSL; every problem is a loud, free UsageFault."""
|
|
61
|
+
properties: dict[str, object] = {}
|
|
62
|
+
required: list[str] = []
|
|
63
|
+
fields = [field.strip() for field in text.split(";") if field.strip()]
|
|
64
|
+
if not fields:
|
|
65
|
+
raise UsageFault(f"--schema-from describes no fields{_HELP}")
|
|
66
|
+
for field in fields:
|
|
67
|
+
name, prop, is_required = _parse_field(field)
|
|
68
|
+
if name in properties:
|
|
69
|
+
raise UsageFault(f"--schema-from names {name!r} more than once{_HELP}")
|
|
70
|
+
properties[name] = prop
|
|
71
|
+
if is_required:
|
|
72
|
+
required.append(name)
|
|
73
|
+
return {
|
|
74
|
+
"type": "object",
|
|
75
|
+
"properties": properties,
|
|
76
|
+
"required": required,
|
|
77
|
+
"additionalProperties": False,
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _parse_field(field: str) -> tuple[str, dict[str, object], bool]:
|
|
82
|
+
name, _space, rest = field.partition(" ")
|
|
83
|
+
if not _NAME.match(name):
|
|
84
|
+
raise UsageFault(f"--schema-from: field names must be identifiers, got {name!r}{_HELP}")
|
|
85
|
+
rest = rest.strip()
|
|
86
|
+
prop, remainder = _parse_type(name, rest)
|
|
87
|
+
prop, remainder, is_required = _parse_constraints(name, prop, remainder)
|
|
88
|
+
if remainder:
|
|
89
|
+
raise UsageFault(f"--schema-from: unexpected {remainder!r} for field {name!r}{_HELP}")
|
|
90
|
+
return name, prop, is_required
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _parse_type(name: str, rest: str) -> tuple[dict[str, object], str]:
|
|
94
|
+
if rest.startswith("enum("):
|
|
95
|
+
close = rest.find(")")
|
|
96
|
+
if close == -1:
|
|
97
|
+
raise UsageFault(f"--schema-from: unclosed enum( for field {name!r}{_HELP}")
|
|
98
|
+
values = [value.strip() for value in rest[len("enum(") : close].split(",")]
|
|
99
|
+
values = [value for value in values if value]
|
|
100
|
+
if not values:
|
|
101
|
+
raise UsageFault(f"--schema-from: enum needs at least one value for {name!r}{_HELP}")
|
|
102
|
+
return {"enum": values}, rest[close + 1 :].strip()
|
|
103
|
+
head, _space, tail = rest.partition(" ")
|
|
104
|
+
simple = _SIMPLE_TYPES.get(head)
|
|
105
|
+
if simple is None:
|
|
106
|
+
offending = head if head else "(nothing)"
|
|
107
|
+
raise UsageFault(f"--schema-from: unexpected {offending!r} for field {name!r}{_HELP}")
|
|
108
|
+
return dict(simple), tail.strip()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _parse_constraints(
|
|
112
|
+
name: str, prop: dict[str, object], remainder: str
|
|
113
|
+
) -> tuple[dict[str, object], str, bool]:
|
|
114
|
+
is_required = True
|
|
115
|
+
kind = prop.get("type")
|
|
116
|
+
while remainder:
|
|
117
|
+
if remainder.startswith("optional"):
|
|
118
|
+
is_required = False
|
|
119
|
+
remainder = remainder[len("optional") :].strip()
|
|
120
|
+
continue
|
|
121
|
+
bound = _BOUND.match(remainder)
|
|
122
|
+
if bound is not None:
|
|
123
|
+
if kind not in ("number", "integer"):
|
|
124
|
+
raise UsageFault(
|
|
125
|
+
f"--schema-from: {bound.group(1)} only applies to number/integer "
|
|
126
|
+
f"(field {name!r} is {kind or 'enum'}){_HELP}"
|
|
127
|
+
)
|
|
128
|
+
key = "minimum" if bound.group(1) == ">=" else "maximum"
|
|
129
|
+
value = bound.group(2)
|
|
130
|
+
prop[key] = int(value) if "." not in value else float(value)
|
|
131
|
+
remainder = remainder[bound.end() :].strip()
|
|
132
|
+
continue
|
|
133
|
+
length = _LENGTH.match(remainder)
|
|
134
|
+
if length is not None:
|
|
135
|
+
if kind != "string":
|
|
136
|
+
raise UsageFault(
|
|
137
|
+
f"--schema-from: {length.group(1)} only applies to string "
|
|
138
|
+
f"(field {name!r} is {kind or 'enum'}){_HELP}"
|
|
139
|
+
)
|
|
140
|
+
prop[length.group(1)] = int(length.group(2))
|
|
141
|
+
remainder = remainder[length.end() :].strip()
|
|
142
|
+
continue
|
|
143
|
+
break # unconsumed — the caller names it
|
|
144
|
+
return prop, remainder, is_required
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""``--tally FIELD``: count a field's values across structured results (pure)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
|
|
11
|
+
__all__ = ["Tally", "explode_record", "render_tally"]
|
|
12
|
+
|
|
13
|
+
_MISSING = "(missing)"
|
|
14
|
+
_TOP_LIVE = 3 # the status line shows the leaders; the final line shows everything
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Tally:
|
|
18
|
+
"""Counts one field's values as structured results land."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, field: str) -> None:
|
|
21
|
+
self.field = field
|
|
22
|
+
self.counts: Counter[str] = Counter()
|
|
23
|
+
|
|
24
|
+
def add(self, record: Mapping[str, object]) -> None:
|
|
25
|
+
value = record.get(self.field, None)
|
|
26
|
+
self.counts[_MISSING if value is None else str(value)] += 1
|
|
27
|
+
|
|
28
|
+
def live_segment(self) -> str:
|
|
29
|
+
return render_tally(self.counts, limit=_TOP_LIVE)
|
|
30
|
+
|
|
31
|
+
def final_line(self) -> str:
|
|
32
|
+
return f"tally: {render_tally(self.counts, limit=None)}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def render_tally(counts: Counter[str], *, limit: int | None) -> str:
|
|
36
|
+
ranked = counts.most_common(limit)
|
|
37
|
+
rendered = " · ".join(f"{value} {count}" for value, count in ranked)
|
|
38
|
+
if limit is not None and len(counts) > limit:
|
|
39
|
+
rendered += " · …"
|
|
40
|
+
return rendered
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def explode_record(record: Mapping[str, object], field: str) -> list[dict[str, object]]:
|
|
44
|
+
"""``--explode``: one row per element of a list-valued field, sibling fields
|
|
45
|
+
copied. Non-lists (including a missing field) pass through as one row."""
|
|
46
|
+
from smartpipe.core.jsontools import as_items
|
|
47
|
+
|
|
48
|
+
value = as_items(record.get(field))
|
|
49
|
+
if value is None:
|
|
50
|
+
return [dict(record)]
|
|
51
|
+
if not value:
|
|
52
|
+
return [] # an empty list is zero rows — nothing to say, honestly
|
|
53
|
+
return [{**record, field: element} for element in value]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Time bucketing for chart/summarize (D38/13, KQL ``bin()``) — pure, fenced.
|
|
2
|
+
|
|
3
|
+
The fence IS the design: timestamps parse as ISO-8601 or epoch
|
|
4
|
+
seconds/milliseconds, nothing else — timestamp-format hell is the swamp KQL
|
|
5
|
+
never had to cross (its ingest normalizes time), and we refuse to cross it
|
|
6
|
+
one format at a time. Labels are UTC.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
|
|
13
|
+
from smartpipe.core.errors import UsageFault
|
|
14
|
+
|
|
15
|
+
__all__ = ["BUCKETS_MENU", "bucket_label", "parse_bucket", "parse_timestamp"]
|
|
16
|
+
|
|
17
|
+
_BUCKETS: dict[str, int] = {
|
|
18
|
+
"1m": 60,
|
|
19
|
+
"5m": 300,
|
|
20
|
+
"15m": 900,
|
|
21
|
+
"1h": 3_600,
|
|
22
|
+
"6h": 21_600,
|
|
23
|
+
"1d": 86_400,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
BUCKETS_MENU = (
|
|
27
|
+
"error: unknown time bucket\n"
|
|
28
|
+
" Buckets: 1m · 5m · 15m · 1h · 6h · 1d\n"
|
|
29
|
+
" Example: smartpipe chart --by-time ts:1h"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
_EPOCH_MILLIS_FLOOR = 1e11 # numbers past this are milliseconds, not seconds
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def parse_bucket(text: str) -> int:
|
|
36
|
+
seconds = _BUCKETS.get(text.strip())
|
|
37
|
+
if seconds is None:
|
|
38
|
+
raise UsageFault(BUCKETS_MENU + f"\n (got: {text!r})")
|
|
39
|
+
return seconds
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_timestamp(value: object) -> float | None:
|
|
43
|
+
"""Epoch seconds from ISO-8601 or epoch numbers; None means unparseable
|
|
44
|
+
(the caller tallies and discloses — silence must never lie)."""
|
|
45
|
+
if isinstance(value, bool):
|
|
46
|
+
return None
|
|
47
|
+
if isinstance(value, (int, float)):
|
|
48
|
+
number = float(value)
|
|
49
|
+
return number / 1000.0 if number >= _EPOCH_MILLIS_FLOOR else number
|
|
50
|
+
if isinstance(value, str):
|
|
51
|
+
text = value.strip().replace("Z", "+00:00")
|
|
52
|
+
try:
|
|
53
|
+
moment = datetime.fromisoformat(text)
|
|
54
|
+
except ValueError:
|
|
55
|
+
return None
|
|
56
|
+
if moment.tzinfo is None:
|
|
57
|
+
moment = moment.replace(tzinfo=UTC) # naive reads as UTC, documented
|
|
58
|
+
return moment.timestamp()
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def bucket_label(epoch: float, bucket_seconds: int) -> str:
|
|
63
|
+
floored = int(epoch // bucket_seconds) * bucket_seconds
|
|
64
|
+
moment = datetime.fromtimestamp(floored, tz=UTC)
|
|
65
|
+
if bucket_seconds >= 86_400:
|
|
66
|
+
return moment.strftime("%Y-%m-%d")
|
|
67
|
+
return moment.strftime("%H:%M")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""``split --by`` grammar (D26/D27): UNIT[:N], parsed deterministically, free."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from smartpipe.core.errors import UsageFault
|
|
9
|
+
|
|
10
|
+
__all__ = ["SplitBy", "parse_by"]
|
|
11
|
+
|
|
12
|
+
Unit = Literal["tokens", "pages", "minutes", "seconds"]
|
|
13
|
+
|
|
14
|
+
_DEFAULTS: dict[Unit, int] = {"tokens": 2_000, "pages": 1, "minutes": 10, "seconds": 600}
|
|
15
|
+
_HELP = "\n Examples: --by pages · --by pages:5 · --by minutes:10 · --by tokens:2000"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class SplitBy:
|
|
20
|
+
unit: Unit
|
|
21
|
+
amount: int
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def slice_seconds(self) -> int:
|
|
25
|
+
"""The duration units, normalized to seconds."""
|
|
26
|
+
assert self.unit in ("minutes", "seconds")
|
|
27
|
+
return self.amount * 60 if self.unit == "minutes" else self.amount
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_by(text: str) -> SplitBy:
|
|
31
|
+
"""``"pages"`` / ``"pages:5"`` / ``"minutes:10"`` → a validated SplitBy."""
|
|
32
|
+
unit_text, colon, amount_text = text.partition(":")
|
|
33
|
+
unit = unit_text.strip()
|
|
34
|
+
if unit not in ("tokens", "pages", "minutes", "seconds"):
|
|
35
|
+
raise UsageFault(
|
|
36
|
+
f"--by wants UNIT or UNIT:N — units: tokens, pages, minutes, seconds{_HELP}"
|
|
37
|
+
)
|
|
38
|
+
if not colon:
|
|
39
|
+
return SplitBy(unit, _DEFAULTS[unit])
|
|
40
|
+
stripped = amount_text.strip()
|
|
41
|
+
if not stripped.isdigit() or int(stripped) < 1:
|
|
42
|
+
raise UsageFault(f"--by {unit}:N wants a positive whole number, got {stripped!r}{_HELP}")
|
|
43
|
+
return SplitBy(unit, int(stripped))
|