smartpipe-cli 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. smartpipe/__init__.py +6 -0
  2. smartpipe/__main__.py +8 -0
  3. smartpipe/assets/probe.png +0 -0
  4. smartpipe/assets/probe.txt +1 -0
  5. smartpipe/assets/probe.wav +0 -0
  6. smartpipe/cli/__init__.py +5 -0
  7. smartpipe/cli/auth_cmd.py +78 -0
  8. smartpipe/cli/cache_cmd.py +60 -0
  9. smartpipe/cli/chart_cmd.py +60 -0
  10. smartpipe/cli/cite_cmd.py +26 -0
  11. smartpipe/cli/cluster_cmd.py +102 -0
  12. smartpipe/cli/completions.py +91 -0
  13. smartpipe/cli/config_cmd.py +234 -0
  14. smartpipe/cli/diff_cmd.py +100 -0
  15. smartpipe/cli/distinct_cmd.py +94 -0
  16. smartpipe/cli/doctor_cmd.py +207 -0
  17. smartpipe/cli/echo_cmd.py +44 -0
  18. smartpipe/cli/embed_cmd.py +80 -0
  19. smartpipe/cli/extend_cmd.py +138 -0
  20. smartpipe/cli/filter_cmd.py +87 -0
  21. smartpipe/cli/getschema_cmd.py +31 -0
  22. smartpipe/cli/input_options.py +113 -0
  23. smartpipe/cli/interrupts.py +92 -0
  24. smartpipe/cli/join_cmd.py +162 -0
  25. smartpipe/cli/map_cmd.py +150 -0
  26. smartpipe/cli/outliers_cmd.py +82 -0
  27. smartpipe/cli/probe_cmd.py +223 -0
  28. smartpipe/cli/reduce_cmd.py +129 -0
  29. smartpipe/cli/root.py +281 -0
  30. smartpipe/cli/run_cmd.py +136 -0
  31. smartpipe/cli/sample_cmd.py +35 -0
  32. smartpipe/cli/schema_cmd.py +75 -0
  33. smartpipe/cli/screens.py +231 -0
  34. smartpipe/cli/sem_file.py +453 -0
  35. smartpipe/cli/sort_cmd.py +33 -0
  36. smartpipe/cli/split_cmd.py +76 -0
  37. smartpipe/cli/summarize_cmd.py +37 -0
  38. smartpipe/cli/top_k_cmd.py +97 -0
  39. smartpipe/cli/usage_cmd.py +66 -0
  40. smartpipe/cli/where_cmd.py +36 -0
  41. smartpipe/config/__init__.py +5 -0
  42. smartpipe/config/credentials.py +118 -0
  43. smartpipe/config/display.py +70 -0
  44. smartpipe/config/doctor.py +58 -0
  45. smartpipe/config/paths.py +38 -0
  46. smartpipe/config/store.py +252 -0
  47. smartpipe/container.py +439 -0
  48. smartpipe/core/__init__.py +5 -0
  49. smartpipe/core/errors.py +57 -0
  50. smartpipe/core/jsontools.py +56 -0
  51. smartpipe/engine/__init__.py +9 -0
  52. smartpipe/engine/aggregate.py +234 -0
  53. smartpipe/engine/blocking.py +44 -0
  54. smartpipe/engine/chart.py +143 -0
  55. smartpipe/engine/chunking.py +161 -0
  56. smartpipe/engine/clustering.py +94 -0
  57. smartpipe/engine/predicate.py +330 -0
  58. smartpipe/engine/prompts.py +601 -0
  59. smartpipe/engine/ranking.py +62 -0
  60. smartpipe/engine/runner.py +175 -0
  61. smartpipe/engine/schema.py +208 -0
  62. smartpipe/engine/schema_dsl.py +144 -0
  63. smartpipe/engine/tally.py +53 -0
  64. smartpipe/engine/timebin.py +67 -0
  65. smartpipe/engine/units.py +43 -0
  66. smartpipe/engine/windows.py +78 -0
  67. smartpipe/io/__init__.py +9 -0
  68. smartpipe/io/diagnostics.py +148 -0
  69. smartpipe/io/inputs.py +44 -0
  70. smartpipe/io/items.py +149 -0
  71. smartpipe/io/leaderboard.py +52 -0
  72. smartpipe/io/metering.py +180 -0
  73. smartpipe/io/progress.py +140 -0
  74. smartpipe/io/readers.py +455 -0
  75. smartpipe/io/text.py +40 -0
  76. smartpipe/io/tty.py +88 -0
  77. smartpipe/io/usage.py +214 -0
  78. smartpipe/io/writers.py +340 -0
  79. smartpipe/models/__init__.py +5 -0
  80. smartpipe/models/anthropic_adapter.py +149 -0
  81. smartpipe/models/base.py +170 -0
  82. smartpipe/models/budget.py +94 -0
  83. smartpipe/models/cache.py +132 -0
  84. smartpipe/models/gemini_native.py +196 -0
  85. smartpipe/models/http_support.py +77 -0
  86. smartpipe/models/jina.py +98 -0
  87. smartpipe/models/local_embed.py +76 -0
  88. smartpipe/models/ollama.py +204 -0
  89. smartpipe/models/openai_codex.py +237 -0
  90. smartpipe/models/openai_compat.py +328 -0
  91. smartpipe/models/openai_oauth.py +366 -0
  92. smartpipe/models/resolve.py +78 -0
  93. smartpipe/models/retry.py +69 -0
  94. smartpipe/models/stt.py +80 -0
  95. smartpipe/models/windows.py +116 -0
  96. smartpipe/parsing/__init__.py +10 -0
  97. smartpipe/parsing/detect.py +178 -0
  98. smartpipe/parsing/extract.py +582 -0
  99. smartpipe/py.typed +0 -0
  100. smartpipe/verbs/__init__.py +5 -0
  101. smartpipe/verbs/chart.py +153 -0
  102. smartpipe/verbs/cluster.py +220 -0
  103. smartpipe/verbs/common.py +468 -0
  104. smartpipe/verbs/convert.py +251 -0
  105. smartpipe/verbs/diff.py +206 -0
  106. smartpipe/verbs/distinct.py +164 -0
  107. smartpipe/verbs/embed.py +166 -0
  108. smartpipe/verbs/extend.py +180 -0
  109. smartpipe/verbs/filter.py +191 -0
  110. smartpipe/verbs/getschema.py +135 -0
  111. smartpipe/verbs/join.py +413 -0
  112. smartpipe/verbs/map.py +315 -0
  113. smartpipe/verbs/outliers.py +119 -0
  114. smartpipe/verbs/reduce.py +428 -0
  115. smartpipe/verbs/sample.py +52 -0
  116. smartpipe/verbs/sortverb.py +63 -0
  117. smartpipe/verbs/split.py +333 -0
  118. smartpipe/verbs/summarize.py +60 -0
  119. smartpipe/verbs/top_k.py +318 -0
  120. smartpipe/verbs/where.py +47 -0
  121. smartpipe_cli-1.3.0.dist-info/METADATA +192 -0
  122. smartpipe_cli-1.3.0.dist-info/RECORD +126 -0
  123. smartpipe_cli-1.3.0.dist-info/WHEEL +4 -0
  124. smartpipe_cli-1.3.0.dist-info/entry_points.txt +3 -0
  125. smartpipe_cli-1.3.0.dist-info/licenses/LICENSE +201 -0
  126. smartpipe_cli-1.3.0.dist-info/licenses/NOTICE +5 -0
@@ -0,0 +1,175 @@
1
+ """Ordered, bounded-concurrency execution — the one primitive every per-item
2
+ verb runs on (plan/architecture.md "Execution engine").
3
+
4
+ Guarantees (all property-tested):
5
+ 1. Order — outcomes yield in input order, always. Grep-shaped tools that reorder
6
+ lines break every downstream diff/paste/log habit.
7
+ 2. Boundedness — at most ``concurrency`` workers in flight; memory O(concurrency)
8
+ regardless of input size, so it streams unbounded input the same as a batch.
9
+ 3. Isolation — a worker raising ``ItemError`` yields ``Skipped`` and the run
10
+ continues; any other exception propagates (it's a bug, crash loudly).
11
+ 4. Accounting — once enough items finish, a majority-failure run halts with
12
+ ``TooManyFailures`` rather than burning the whole input on a broken config.
13
+
14
+ The worker is injected (a first-class async function), so this module does no
15
+ I/O of its own and orders purely by arrival.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import asyncio
21
+ from dataclasses import dataclass
22
+ from typing import TYPE_CHECKING, Generic, TypeVar
23
+
24
+ from smartpipe.core.errors import ItemError, TooManyFailures
25
+
26
+ if TYPE_CHECKING:
27
+ from collections.abc import AsyncIterator, Awaitable, Callable
28
+
29
+ from smartpipe.io.items import Item, ItemSource
30
+
31
+ __all__ = [
32
+ "Done",
33
+ "FailurePolicy",
34
+ "ItemOutcome",
35
+ "Skipped",
36
+ "run_ordered",
37
+ "should_halt",
38
+ "should_halt_consecutive",
39
+ ]
40
+
41
+ R = TypeVar("R")
42
+
43
+
44
+ @dataclass(frozen=True, slots=True)
45
+ class Done(Generic[R]):
46
+ index: int
47
+ value: R
48
+
49
+
50
+ @dataclass(frozen=True, slots=True)
51
+ class Skipped:
52
+ index: int
53
+ reason: str
54
+ source: ItemSource
55
+
56
+
57
+ ItemOutcome = Done[R] | Skipped
58
+
59
+
60
+ @dataclass(frozen=True, slots=True)
61
+ class FailurePolicy:
62
+ halt_ratio: float = 0.5
63
+ min_sample: int = 20
64
+ consecutive_limit: int = 5 # D18: a doomed run must not wait for the ratio
65
+
66
+
67
+ def should_halt(policy: FailurePolicy, *, total: int, skipped: int) -> bool:
68
+ """True once enough items have finished and a majority of them failed.
69
+ ``min_sample`` prevents a 3-item pipe halting on 2 flukes."""
70
+ return total >= policy.min_sample and skipped > total * policy.halt_ratio
71
+
72
+
73
+ def should_halt_consecutive(policy: FailurePolicy, *, succeeded: bool, consecutive: int) -> bool:
74
+ """D18's cost guardrail: N consecutive failures with zero successes *ever* means
75
+ the run was doomed from item 1 — stop paying. Any success disarms this rule
76
+ permanently (a working run with a bad patch of input must not die early)."""
77
+ return not succeeded and consecutive >= policy.consecutive_limit
78
+
79
+
80
+ async def run_ordered(
81
+ items: AsyncIterator[Item],
82
+ worker: Callable[[Item], Awaitable[R]],
83
+ *,
84
+ concurrency: int,
85
+ failure_policy: FailurePolicy,
86
+ stop: asyncio.Event | None = None,
87
+ ) -> AsyncIterator[ItemOutcome[R]]:
88
+ """``stop`` (set by the interrupt shell) halts *intake*: no new workers spawn,
89
+ but everything already in flight completes and is emitted in order — the drain
90
+ contract of ux.md §12.
91
+
92
+ Intake runs as its OWN task so that waiting for the next input item never
93
+ blocks the emission of already-completed outcomes — the streaming property
94
+ (a live stream can pause mid-flow; results must still come out).
95
+ """
96
+ item_iter = aiter(items)
97
+ pending: dict[int, asyncio.Task[ItemOutcome[R]]] = {}
98
+ slots = asyncio.Semaphore(concurrency)
99
+ progressed = asyncio.Event() # set whenever intake adds a task or finishes
100
+ intake_done = False
101
+ next_to_emit = 0
102
+ total = 0
103
+ skipped = 0
104
+ consecutive = 0
105
+ succeeded = False
106
+
107
+ def stopping() -> bool:
108
+ return stop is not None and stop.is_set()
109
+
110
+ async def intake() -> None:
111
+ nonlocal intake_done
112
+ index = 0
113
+ try:
114
+ while not stopping():
115
+ await slots.acquire()
116
+ if stopping(): # woke up into a drain — don't start new work
117
+ slots.release()
118
+ break
119
+ try:
120
+ item = await anext(item_iter)
121
+ except StopAsyncIteration:
122
+ slots.release()
123
+ break
124
+ pending[index] = asyncio.create_task(_run_one(worker, item))
125
+ index += 1
126
+ progressed.set()
127
+ finally:
128
+ intake_done = True
129
+ progressed.set()
130
+
131
+ intake_task = asyncio.create_task(intake())
132
+ try:
133
+ while True:
134
+ task = pending.get(next_to_emit)
135
+ if task is None:
136
+ if intake_done and not pending:
137
+ return
138
+ progressed.clear()
139
+ # re-check before sleeping: intake may have progressed between the
140
+ # get() above and the clear() — a real race, so the branch can't be
141
+ # hit deterministically in a test; excluded rather than pretended at.
142
+ if next_to_emit in pending or (intake_done and not pending): # pragma: no cover
143
+ continue # pragma: no cover
144
+ await progressed.wait()
145
+ continue
146
+ outcome = await task
147
+ del pending[next_to_emit]
148
+ slots.release()
149
+ next_to_emit += 1
150
+ yield outcome
151
+ total += 1
152
+ if isinstance(outcome, Skipped):
153
+ skipped += 1
154
+ consecutive += 1
155
+ if should_halt(failure_policy, total=total, skipped=skipped):
156
+ raise TooManyFailures(skipped, total, outcome.reason)
157
+ if should_halt_consecutive(
158
+ failure_policy, succeeded=succeeded, consecutive=consecutive
159
+ ):
160
+ raise TooManyFailures(skipped, total, outcome.reason)
161
+ else:
162
+ consecutive = 0
163
+ succeeded = True
164
+ finally:
165
+ intake_task.cancel()
166
+ for task in pending.values():
167
+ task.cancel()
168
+ await asyncio.gather(intake_task, *pending.values(), return_exceptions=True)
169
+
170
+
171
+ async def _run_one(worker: Callable[[Item], Awaitable[R]], item: Item) -> ItemOutcome[R]:
172
+ try:
173
+ return Done(item.source.index, await worker(item))
174
+ except ItemError as exc:
175
+ return Skipped(item.source.index, str(exc), item.source)
@@ -0,0 +1,208 @@
1
+ """Structured-output schemas (plan/decisions.md D07): shorthand synthesis,
2
+ JSON-Schema loading, and validate-with-light-coercion of a model reply.
3
+
4
+ ``validate_and_coerce`` raises ``ItemError`` on any failure, with a message that
5
+ names the problem — the ``map`` verb feeds that message back to the model as the
6
+ single repair retry, so the message is repair context, not just a log line.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import re
13
+ from typing import TYPE_CHECKING
14
+
15
+ from smartpipe.core.errors import ItemError, SetupFault
16
+ from smartpipe.core.jsontools import as_items, as_record, as_str
17
+
18
+ if TYPE_CHECKING:
19
+ from collections.abc import Mapping, Sequence
20
+ from pathlib import Path
21
+
22
+ __all__ = [
23
+ "is_strict_compatible",
24
+ "load_schema",
25
+ "parse_schema_draft",
26
+ "shorthand_to_schema",
27
+ "validate_and_coerce",
28
+ ]
29
+
30
+ _FENCE = re.compile(r"^```[A-Za-z0-9]*\n?|\n?```$")
31
+
32
+
33
+ def shorthand_to_schema(
34
+ fields: Sequence[str],
35
+ *,
36
+ descriptions: Mapping[str, str] | None = None,
37
+ types: Mapping[str, Mapping[str, object]] | None = None,
38
+ ) -> dict[str, object]:
39
+ """Turn ``{vendor, total}`` fields into a strict JSON Schema. Inline types
40
+ (D37) and rung-2 descriptions (D22) ride each property; a fully-typed group
41
+ regains strict mode (every property carries a type)."""
42
+ notes = descriptions or {}
43
+ typed = types or {}
44
+
45
+ def _property(field: str) -> dict[str, object]:
46
+ prop: dict[str, object] = dict(typed.get(field, {}))
47
+ if field in notes:
48
+ prop["description"] = notes[field]
49
+ return prop
50
+
51
+ properties: dict[str, object] = {field: _property(field) for field in fields}
52
+ return {
53
+ "type": "object",
54
+ "properties": properties,
55
+ "required": list(fields),
56
+ "additionalProperties": False,
57
+ }
58
+
59
+
60
+ def is_strict_compatible(schema: Mapping[str, object]) -> bool:
61
+ """Would OpenAI/Mistral ``strict: true`` json_schema mode accept this schema?
62
+
63
+ Strict mode demands, at every object layer: every property listed in
64
+ ``required`` and ``additionalProperties: false``. Brace-shorthand schemas
65
+ qualify by construction; a user ``--schema`` with optional fields does not —
66
+ claiming strict for it draws a 400 and skips items for the wrong reason.
67
+ """
68
+ if _looks_like_object(schema):
69
+ if schema.get("additionalProperties") is not False:
70
+ return False
71
+ properties = as_record(schema.get("properties")) or {}
72
+ required = as_items(schema.get("required")) or ()
73
+ required_names = {name for name in required if isinstance(name, str)}
74
+ if not set(properties) <= required_names:
75
+ return False
76
+ for value in properties.values():
77
+ child = as_record(value)
78
+ # live-caught (2026-07-05): strict mode also demands a 'type' per
79
+ # property — an untyped {} (the brace shorthand) draws the same 400
80
+ if child is None or ("type" not in child and "enum" not in child):
81
+ return False
82
+ if not is_strict_compatible(child):
83
+ return False
84
+ items = as_record(schema.get("items"))
85
+ return items is None or is_strict_compatible(items)
86
+
87
+
88
+ def _looks_like_object(schema: Mapping[str, object]) -> bool:
89
+ return schema.get("type") == "object" or as_record(schema.get("properties")) is not None
90
+
91
+
92
+ def load_schema(path: Path) -> dict[str, object]:
93
+ if not path.exists():
94
+ raise SetupFault(
95
+ f"error: no schema file at {path}\n Check the --schema path and try again."
96
+ )
97
+ try:
98
+ loaded = json.loads(path.read_text(encoding="utf-8"))
99
+ except json.JSONDecodeError as exc:
100
+ raise SetupFault(
101
+ f"error: {path} isn't valid JSON\n {exc}\n A --schema file must be a JSON Schema."
102
+ ) from exc
103
+ record = as_record(loaded)
104
+ if record is None:
105
+ raise SetupFault(
106
+ f"error: {path} isn't a JSON Schema\n The top level must be a JSON object."
107
+ )
108
+ return dict(record)
109
+
110
+
111
+ def parse_schema_draft(reply: str) -> dict[str, object]:
112
+ """Rung 4 (D22): a model-drafted schema, validated against the JSON-Schema
113
+ meta-schema AND our loader rules — ``ItemError`` names what's wrong (repair
114
+ context). An invalid draft must never reach stdout."""
115
+ import jsonschema
116
+
117
+ record = as_record(_extract_json(reply)) # ItemError when there's no JSON at all
118
+ if record is None:
119
+ raise ItemError("the draft isn't a JSON object")
120
+ candidate = dict(record)
121
+ try:
122
+ jsonschema.Draft202012Validator.check_schema(candidate)
123
+ except jsonschema.SchemaError as exc:
124
+ raise ItemError(f"not a valid JSON Schema: {exc.message}") from exc
125
+ return candidate
126
+
127
+
128
+ def validate_and_coerce(reply: str, schema: Mapping[str, object]) -> dict[str, object]:
129
+ import jsonschema # function-local: --help must not pay for the validator stack
130
+
131
+ record = as_record(_extract_json(reply))
132
+ if record is None:
133
+ raise ItemError("model returned JSON but not an object")
134
+ coerced = _coerce(record, schema)
135
+ trimmed = _drop_extra(coerced, schema)
136
+ try:
137
+ jsonschema.validate(trimmed, dict(schema))
138
+ except jsonschema.ValidationError as exc:
139
+ raise ItemError(f"output does not match the schema: {exc.message}") from exc
140
+ return trimmed
141
+
142
+
143
+ def _extract_json(reply: str) -> object:
144
+ text = reply.strip()
145
+ if text.startswith("```"):
146
+ text = _FENCE.sub("", text).strip()
147
+ for candidate in _json_candidates(text):
148
+ try:
149
+ return json.loads(candidate)
150
+ except json.JSONDecodeError:
151
+ continue
152
+ raise ItemError("model did not return valid JSON")
153
+
154
+
155
+ def _json_candidates(text: str) -> tuple[str, ...]:
156
+ start = text.find("{")
157
+ end = text.rfind("}")
158
+ if start != -1 and end > start:
159
+ return (text, text[start : end + 1])
160
+ return (text,)
161
+
162
+
163
+ def _coerce(record: Mapping[str, object], schema: Mapping[str, object]) -> dict[str, object]:
164
+ properties = as_record(schema.get("properties")) or {}
165
+ return {
166
+ key: _coerce_scalar(value, as_record(properties.get(key))) for key, value in record.items()
167
+ }
168
+
169
+
170
+ def _coerce_scalar(value: object, prop_schema: Mapping[str, object] | None) -> object:
171
+ if prop_schema is None or not isinstance(value, str):
172
+ return value
173
+ text = value.strip()
174
+ match as_str(prop_schema.get("type")):
175
+ case "integer":
176
+ return int(text) if _is_int(text) else value
177
+ case "number":
178
+ return float(text) if _is_float(text) else value
179
+ case "boolean":
180
+ return _as_bool(text, fallback=value)
181
+ case "null":
182
+ return None if text == "null" else value
183
+ case _:
184
+ return value
185
+
186
+
187
+ def _drop_extra(record: dict[str, object], schema: Mapping[str, object]) -> dict[str, object]:
188
+ if schema.get("additionalProperties") is not False:
189
+ return record
190
+ allowed = as_record(schema.get("properties")) or {}
191
+ return {key: value for key, value in record.items() if key in allowed}
192
+
193
+
194
+ def _is_int(text: str) -> bool:
195
+ return bool(re.fullmatch(r"[+-]?\d+", text))
196
+
197
+
198
+ def _is_float(text: str) -> bool:
199
+ return bool(re.fullmatch(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?", text))
200
+
201
+
202
+ def _as_bool(text: str, *, fallback: object) -> object:
203
+ lowered = text.lower()
204
+ if lowered == "true":
205
+ return True
206
+ if lowered == "false":
207
+ return False
208
+ return fallback
@@ -0,0 +1,144 @@
1
+ """Rung 3 of the schema ladder (D22): a deterministic mini-DSL → JSON Schema.
2
+
3
+ ``vendor string; total number >= 0; status enum(paid, unpaid)`` — parsed with
4
+ zero model calls and zero I/O, so a typo fails at argv time, before anything
5
+ could cost money (D18 applied to schemas). Everything richer than this grammar
6
+ belongs in a ``--schema`` file; the error screens say so.
7
+
8
+ Grammar (pinned in ux.md):
9
+ fields ::= field (";" field)*
10
+ field ::= name type constraint*
11
+ type ::= string | number | integer | boolean | enum(a, b, …)
12
+ | string[] | number[]
13
+ constraint ::= ">=" N | "<=" N | minLength=N | maxLength=N | optional
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import re
19
+
20
+ from smartpipe.core.errors import UsageFault
21
+
22
+ __all__ = ["TYPE_MENU", "dsl_to_schema", "type_token"]
23
+
24
+ _NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
25
+ _HELP = (
26
+ "\n Types: string · number · integer · boolean · enum(a, b, …) · string[] · number[]"
27
+ "\n Constraints: >= N · <= N · minLength=N · maxLength=N · optional"
28
+ )
29
+
30
+ _SIMPLE_TYPES: dict[str, dict[str, object]] = {
31
+ "string": {"type": "string"},
32
+ "number": {"type": "number"},
33
+ "integer": {"type": "integer"},
34
+ "boolean": {"type": "boolean"},
35
+ "string[]": {"type": "array", "items": {"type": "string"}},
36
+ "number[]": {"type": "array", "items": {"type": "number"}},
37
+ }
38
+
39
+ _BOUND = re.compile(r"(>=|<=)\s*(-?\d+(?:\.\d+)?)")
40
+ _LENGTH = re.compile(r"(minLength|maxLength)=(\d+)")
41
+
42
+
43
+ TYPE_MENU = "string · number · integer · boolean · enum(a, b, …) · string[] · number[]"
44
+
45
+
46
+ def type_token(token: str) -> dict[str, object] | None:
47
+ """One type token → a property dict, or None when it isn't a type.
48
+
49
+ Shared vocabulary with the braces (D37): one grammar, two homes."""
50
+ simple = _SIMPLE_TYPES.get(token)
51
+ if simple is not None:
52
+ return dict(simple)
53
+ if token.startswith("enum(") and token.endswith(")"):
54
+ values = [value.strip() for value in token[5:-1].split(",") if value.strip()]
55
+ return {"enum": values} if values else None
56
+ return None
57
+
58
+
59
+ def dsl_to_schema(text: str) -> dict[str, object]:
60
+ """Parse the ``--schema-from`` DSL; every problem is a loud, free UsageFault."""
61
+ properties: dict[str, object] = {}
62
+ required: list[str] = []
63
+ fields = [field.strip() for field in text.split(";") if field.strip()]
64
+ if not fields:
65
+ raise UsageFault(f"--schema-from describes no fields{_HELP}")
66
+ for field in fields:
67
+ name, prop, is_required = _parse_field(field)
68
+ if name in properties:
69
+ raise UsageFault(f"--schema-from names {name!r} more than once{_HELP}")
70
+ properties[name] = prop
71
+ if is_required:
72
+ required.append(name)
73
+ return {
74
+ "type": "object",
75
+ "properties": properties,
76
+ "required": required,
77
+ "additionalProperties": False,
78
+ }
79
+
80
+
81
+ def _parse_field(field: str) -> tuple[str, dict[str, object], bool]:
82
+ name, _space, rest = field.partition(" ")
83
+ if not _NAME.match(name):
84
+ raise UsageFault(f"--schema-from: field names must be identifiers, got {name!r}{_HELP}")
85
+ rest = rest.strip()
86
+ prop, remainder = _parse_type(name, rest)
87
+ prop, remainder, is_required = _parse_constraints(name, prop, remainder)
88
+ if remainder:
89
+ raise UsageFault(f"--schema-from: unexpected {remainder!r} for field {name!r}{_HELP}")
90
+ return name, prop, is_required
91
+
92
+
93
+ def _parse_type(name: str, rest: str) -> tuple[dict[str, object], str]:
94
+ if rest.startswith("enum("):
95
+ close = rest.find(")")
96
+ if close == -1:
97
+ raise UsageFault(f"--schema-from: unclosed enum( for field {name!r}{_HELP}")
98
+ values = [value.strip() for value in rest[len("enum(") : close].split(",")]
99
+ values = [value for value in values if value]
100
+ if not values:
101
+ raise UsageFault(f"--schema-from: enum needs at least one value for {name!r}{_HELP}")
102
+ return {"enum": values}, rest[close + 1 :].strip()
103
+ head, _space, tail = rest.partition(" ")
104
+ simple = _SIMPLE_TYPES.get(head)
105
+ if simple is None:
106
+ offending = head if head else "(nothing)"
107
+ raise UsageFault(f"--schema-from: unexpected {offending!r} for field {name!r}{_HELP}")
108
+ return dict(simple), tail.strip()
109
+
110
+
111
+ def _parse_constraints(
112
+ name: str, prop: dict[str, object], remainder: str
113
+ ) -> tuple[dict[str, object], str, bool]:
114
+ is_required = True
115
+ kind = prop.get("type")
116
+ while remainder:
117
+ if remainder.startswith("optional"):
118
+ is_required = False
119
+ remainder = remainder[len("optional") :].strip()
120
+ continue
121
+ bound = _BOUND.match(remainder)
122
+ if bound is not None:
123
+ if kind not in ("number", "integer"):
124
+ raise UsageFault(
125
+ f"--schema-from: {bound.group(1)} only applies to number/integer "
126
+ f"(field {name!r} is {kind or 'enum'}){_HELP}"
127
+ )
128
+ key = "minimum" if bound.group(1) == ">=" else "maximum"
129
+ value = bound.group(2)
130
+ prop[key] = int(value) if "." not in value else float(value)
131
+ remainder = remainder[bound.end() :].strip()
132
+ continue
133
+ length = _LENGTH.match(remainder)
134
+ if length is not None:
135
+ if kind != "string":
136
+ raise UsageFault(
137
+ f"--schema-from: {length.group(1)} only applies to string "
138
+ f"(field {name!r} is {kind or 'enum'}){_HELP}"
139
+ )
140
+ prop[length.group(1)] = int(length.group(2))
141
+ remainder = remainder[length.end() :].strip()
142
+ continue
143
+ break # unconsumed — the caller names it
144
+ return prop, remainder, is_required
@@ -0,0 +1,53 @@
1
+ """``--tally FIELD``: count a field's values across structured results (pure)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter
6
+ from typing import TYPE_CHECKING
7
+
8
+ if TYPE_CHECKING:
9
+ from collections.abc import Mapping
10
+
11
+ __all__ = ["Tally", "explode_record", "render_tally"]
12
+
13
+ _MISSING = "(missing)"
14
+ _TOP_LIVE = 3 # the status line shows the leaders; the final line shows everything
15
+
16
+
17
+ class Tally:
18
+ """Counts one field's values as structured results land."""
19
+
20
+ def __init__(self, field: str) -> None:
21
+ self.field = field
22
+ self.counts: Counter[str] = Counter()
23
+
24
+ def add(self, record: Mapping[str, object]) -> None:
25
+ value = record.get(self.field, None)
26
+ self.counts[_MISSING if value is None else str(value)] += 1
27
+
28
+ def live_segment(self) -> str:
29
+ return render_tally(self.counts, limit=_TOP_LIVE)
30
+
31
+ def final_line(self) -> str:
32
+ return f"tally: {render_tally(self.counts, limit=None)}"
33
+
34
+
35
+ def render_tally(counts: Counter[str], *, limit: int | None) -> str:
36
+ ranked = counts.most_common(limit)
37
+ rendered = " · ".join(f"{value} {count}" for value, count in ranked)
38
+ if limit is not None and len(counts) > limit:
39
+ rendered += " · …"
40
+ return rendered
41
+
42
+
43
+ def explode_record(record: Mapping[str, object], field: str) -> list[dict[str, object]]:
44
+ """``--explode``: one row per element of a list-valued field, sibling fields
45
+ copied. Non-lists (including a missing field) pass through as one row."""
46
+ from smartpipe.core.jsontools import as_items
47
+
48
+ value = as_items(record.get(field))
49
+ if value is None:
50
+ return [dict(record)]
51
+ if not value:
52
+ return [] # an empty list is zero rows — nothing to say, honestly
53
+ return [{**record, field: element} for element in value]
@@ -0,0 +1,67 @@
1
+ """Time bucketing for chart/summarize (D38/13, KQL ``bin()``) — pure, fenced.
2
+
3
+ The fence IS the design: timestamps parse as ISO-8601 or epoch
4
+ seconds/milliseconds, nothing else — timestamp-format hell is the swamp KQL
5
+ never had to cross (its ingest normalizes time), and we refuse to cross it
6
+ one format at a time. Labels are UTC.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from datetime import UTC, datetime
12
+
13
+ from smartpipe.core.errors import UsageFault
14
+
15
+ __all__ = ["BUCKETS_MENU", "bucket_label", "parse_bucket", "parse_timestamp"]
16
+
17
+ _BUCKETS: dict[str, int] = {
18
+ "1m": 60,
19
+ "5m": 300,
20
+ "15m": 900,
21
+ "1h": 3_600,
22
+ "6h": 21_600,
23
+ "1d": 86_400,
24
+ }
25
+
26
+ BUCKETS_MENU = (
27
+ "error: unknown time bucket\n"
28
+ " Buckets: 1m · 5m · 15m · 1h · 6h · 1d\n"
29
+ " Example: smartpipe chart --by-time ts:1h"
30
+ )
31
+
32
+ _EPOCH_MILLIS_FLOOR = 1e11 # numbers past this are milliseconds, not seconds
33
+
34
+
35
+ def parse_bucket(text: str) -> int:
36
+ seconds = _BUCKETS.get(text.strip())
37
+ if seconds is None:
38
+ raise UsageFault(BUCKETS_MENU + f"\n (got: {text!r})")
39
+ return seconds
40
+
41
+
42
+ def parse_timestamp(value: object) -> float | None:
43
+ """Epoch seconds from ISO-8601 or epoch numbers; None means unparseable
44
+ (the caller tallies and discloses — silence must never lie)."""
45
+ if isinstance(value, bool):
46
+ return None
47
+ if isinstance(value, (int, float)):
48
+ number = float(value)
49
+ return number / 1000.0 if number >= _EPOCH_MILLIS_FLOOR else number
50
+ if isinstance(value, str):
51
+ text = value.strip().replace("Z", "+00:00")
52
+ try:
53
+ moment = datetime.fromisoformat(text)
54
+ except ValueError:
55
+ return None
56
+ if moment.tzinfo is None:
57
+ moment = moment.replace(tzinfo=UTC) # naive reads as UTC, documented
58
+ return moment.timestamp()
59
+ return None
60
+
61
+
62
+ def bucket_label(epoch: float, bucket_seconds: int) -> str:
63
+ floored = int(epoch // bucket_seconds) * bucket_seconds
64
+ moment = datetime.fromtimestamp(floored, tz=UTC)
65
+ if bucket_seconds >= 86_400:
66
+ return moment.strftime("%Y-%m-%d")
67
+ return moment.strftime("%H:%M")
@@ -0,0 +1,43 @@
1
+ """``split --by`` grammar (D26/D27): UNIT[:N], parsed deterministically, free."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Literal
7
+
8
+ from smartpipe.core.errors import UsageFault
9
+
10
+ __all__ = ["SplitBy", "parse_by"]
11
+
12
+ Unit = Literal["tokens", "pages", "minutes", "seconds"]
13
+
14
+ _DEFAULTS: dict[Unit, int] = {"tokens": 2_000, "pages": 1, "minutes": 10, "seconds": 600}
15
+ _HELP = "\n Examples: --by pages · --by pages:5 · --by minutes:10 · --by tokens:2000"
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class SplitBy:
20
+ unit: Unit
21
+ amount: int
22
+
23
+ @property
24
+ def slice_seconds(self) -> int:
25
+ """The duration units, normalized to seconds."""
26
+ assert self.unit in ("minutes", "seconds")
27
+ return self.amount * 60 if self.unit == "minutes" else self.amount
28
+
29
+
30
+ def parse_by(text: str) -> SplitBy:
31
+ """``"pages"`` / ``"pages:5"`` / ``"minutes:10"`` → a validated SplitBy."""
32
+ unit_text, colon, amount_text = text.partition(":")
33
+ unit = unit_text.strip()
34
+ if unit not in ("tokens", "pages", "minutes", "seconds"):
35
+ raise UsageFault(
36
+ f"--by wants UNIT or UNIT:N — units: tokens, pages, minutes, seconds{_HELP}"
37
+ )
38
+ if not colon:
39
+ return SplitBy(unit, _DEFAULTS[unit])
40
+ stripped = amount_text.strip()
41
+ if not stripped.isdigit() or int(stripped) < 1:
42
+ raise UsageFault(f"--by {unit}:N wants a positive whole number, got {stripped!r}{_HELP}")
43
+ return SplitBy(unit, int(stripped))