parseforge 0.2.12__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parseforge-0.2.12/parseforge.egg-info → parseforge-0.3.0}/PKG-INFO +65 -20
- {parseforge-0.2.12 → parseforge-0.3.0}/README.md +45 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/SPEC.md +2 -2
- parseforge-0.3.0/parseforge/__init__.py +96 -0
- parseforge-0.3.0/parseforge/api.py +127 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/cli/main.py +2 -1
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/generation.py +0 -2
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/llm.py +0 -1
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/anthropic.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/azure.py +1 -16
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/bedrock.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/cerebras.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/cohere.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/deepseek.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/fireworks.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/gemini.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/groq.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/mistral.py +1 -11
- parseforge-0.3.0/parseforge/naming/providers/models.py +17 -0
- parseforge-0.3.0/parseforge/naming/providers/models.yaml +30 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/moonshot.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/oci.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/openai.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/openrouter.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/perplexity.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/together.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/vertexai.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/xai.py +1 -11
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/pipeline.py +25 -3
- {parseforge-0.2.12 → parseforge-0.3.0/parseforge.egg-info}/PKG-INFO +65 -20
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge.egg-info/SOURCES.txt +1 -1
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge.egg-info/requires.txt +19 -19
- {parseforge-0.2.12 → parseforge-0.3.0}/pyproject.toml +26 -20
- parseforge-0.2.12/parseforge/__init__.py +0 -1
- parseforge-0.2.12/parseforge/naming/providers/cost.py +0 -29
- parseforge-0.2.12/parseforge/naming/providers/models.py +0 -26
- parseforge-0.2.12/parseforge/naming/providers/models.yaml +0 -182
- {parseforge-0.2.12 → parseforge-0.3.0}/LICENSE +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/MANIFEST.in +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/cli/__init__.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/cli/config.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/drift.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/integration.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/__init__.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/assemble.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/cache.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/prompts.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/prompts.yaml +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/__init__.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/errors.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/providers/text.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/naming/resolver.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/paths.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/promotion.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/sampling/__init__.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/sampling/backends/__init__.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/sampling/backends/netmiko.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/sampling/core.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge/validation.py +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge.egg-info/dependency_links.txt +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge.egg-info/entry_points.txt +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/parseforge.egg-info/top_level.txt +0 -0
- {parseforge-0.2.12 → parseforge-0.3.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parseforge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: LLM-driven pipeline that forges, validates, and promotes TextFSM templates from network CLI output
|
|
5
5
|
Author-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
|
|
6
6
|
Maintainer-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
|
|
@@ -26,61 +26,61 @@ License-File: LICENSE
|
|
|
26
26
|
Requires-Dist: click>=8.1
|
|
27
27
|
Requires-Dist: PyYAML>=6.0
|
|
28
28
|
Requires-Dist: textfsm>=1.1.0
|
|
29
|
-
Requires-Dist: textfsm-ai>=0.
|
|
29
|
+
Requires-Dist: textfsm-ai>=0.7.1
|
|
30
30
|
Provides-Extra: anthropic
|
|
31
31
|
Requires-Dist: anthropic>=0.40.0; extra == "anthropic"
|
|
32
|
-
Requires-Dist: textfsm-ai[anthropic]>=0.
|
|
32
|
+
Requires-Dist: textfsm-ai[anthropic]>=0.7.1; extra == "anthropic"
|
|
33
33
|
Provides-Extra: deepseek
|
|
34
34
|
Requires-Dist: openai>=1.0.0; extra == "deepseek"
|
|
35
|
-
Requires-Dist: textfsm-ai[deepseek]>=0.
|
|
35
|
+
Requires-Dist: textfsm-ai[deepseek]>=0.7.1; extra == "deepseek"
|
|
36
36
|
Provides-Extra: openai
|
|
37
37
|
Requires-Dist: openai>=1.0.0; extra == "openai"
|
|
38
|
-
Requires-Dist: textfsm-ai[openai]>=0.
|
|
38
|
+
Requires-Dist: textfsm-ai[openai]>=0.7.1; extra == "openai"
|
|
39
39
|
Provides-Extra: groq
|
|
40
40
|
Requires-Dist: openai>=1.0.0; extra == "groq"
|
|
41
|
-
Requires-Dist: textfsm-ai[groq]>=0.
|
|
41
|
+
Requires-Dist: textfsm-ai[groq]>=0.7.1; extra == "groq"
|
|
42
42
|
Provides-Extra: xai
|
|
43
43
|
Requires-Dist: openai>=1.0.0; extra == "xai"
|
|
44
|
-
Requires-Dist: textfsm-ai[xai]>=0.
|
|
44
|
+
Requires-Dist: textfsm-ai[xai]>=0.7.1; extra == "xai"
|
|
45
45
|
Provides-Extra: together
|
|
46
46
|
Requires-Dist: openai>=1.0.0; extra == "together"
|
|
47
|
-
Requires-Dist: textfsm-ai[together]>=0.
|
|
47
|
+
Requires-Dist: textfsm-ai[together]>=0.7.1; extra == "together"
|
|
48
48
|
Provides-Extra: fireworks
|
|
49
49
|
Requires-Dist: openai>=1.0.0; extra == "fireworks"
|
|
50
|
-
Requires-Dist: textfsm-ai[fireworks]>=0.
|
|
50
|
+
Requires-Dist: textfsm-ai[fireworks]>=0.7.1; extra == "fireworks"
|
|
51
51
|
Provides-Extra: perplexity
|
|
52
52
|
Requires-Dist: openai>=1.0.0; extra == "perplexity"
|
|
53
|
-
Requires-Dist: textfsm-ai[perplexity]>=0.
|
|
53
|
+
Requires-Dist: textfsm-ai[perplexity]>=0.7.1; extra == "perplexity"
|
|
54
54
|
Provides-Extra: openrouter
|
|
55
55
|
Requires-Dist: openai>=1.0.0; extra == "openrouter"
|
|
56
|
-
Requires-Dist: textfsm-ai[openrouter]>=0.
|
|
56
|
+
Requires-Dist: textfsm-ai[openrouter]>=0.7.1; extra == "openrouter"
|
|
57
57
|
Provides-Extra: moonshot
|
|
58
58
|
Requires-Dist: openai>=1.0.0; extra == "moonshot"
|
|
59
|
-
Requires-Dist: textfsm-ai[moonshot]>=0.
|
|
59
|
+
Requires-Dist: textfsm-ai[moonshot]>=0.7.1; extra == "moonshot"
|
|
60
60
|
Provides-Extra: cerebras
|
|
61
61
|
Requires-Dist: openai>=1.0.0; extra == "cerebras"
|
|
62
|
-
Requires-Dist: textfsm-ai[cerebras]>=0.
|
|
62
|
+
Requires-Dist: textfsm-ai[cerebras]>=0.7.1; extra == "cerebras"
|
|
63
63
|
Provides-Extra: mistral
|
|
64
64
|
Requires-Dist: mistralai==1.10.0; extra == "mistral"
|
|
65
|
-
Requires-Dist: textfsm-ai[mistral]>=0.
|
|
65
|
+
Requires-Dist: textfsm-ai[mistral]>=0.7.1; extra == "mistral"
|
|
66
66
|
Provides-Extra: cohere
|
|
67
67
|
Requires-Dist: cohere==5.21.1; extra == "cohere"
|
|
68
|
-
Requires-Dist: textfsm-ai[cohere]>=0.
|
|
68
|
+
Requires-Dist: textfsm-ai[cohere]>=0.7.1; extra == "cohere"
|
|
69
69
|
Provides-Extra: azure
|
|
70
70
|
Requires-Dist: azure-ai-inference>=1.0.0b9; extra == "azure"
|
|
71
|
-
Requires-Dist: textfsm-ai[azure]>=0.
|
|
71
|
+
Requires-Dist: textfsm-ai[azure]>=0.7.1; extra == "azure"
|
|
72
72
|
Provides-Extra: gemini
|
|
73
73
|
Requires-Dist: google-genai>=0.2.0; extra == "gemini"
|
|
74
|
-
Requires-Dist: textfsm-ai[gemini]>=0.
|
|
74
|
+
Requires-Dist: textfsm-ai[gemini]>=0.7.1; extra == "gemini"
|
|
75
75
|
Provides-Extra: vertexai
|
|
76
76
|
Requires-Dist: google-genai>=0.2.0; extra == "vertexai"
|
|
77
|
-
Requires-Dist: textfsm-ai[vertexai]>=0.
|
|
77
|
+
Requires-Dist: textfsm-ai[vertexai]>=0.7.1; extra == "vertexai"
|
|
78
78
|
Provides-Extra: bedrock
|
|
79
79
|
Requires-Dist: boto3==1.42.97; extra == "bedrock"
|
|
80
|
-
Requires-Dist: textfsm-ai[bedrock]>=0.
|
|
80
|
+
Requires-Dist: textfsm-ai[bedrock]>=0.7.1; extra == "bedrock"
|
|
81
81
|
Provides-Extra: oci
|
|
82
82
|
Requires-Dist: oci==2.182.0; extra == "oci"
|
|
83
|
-
Requires-Dist: textfsm-ai[oci]>=0.
|
|
83
|
+
Requires-Dist: textfsm-ai[oci]>=0.7.1; extra == "oci"
|
|
84
84
|
Provides-Extra: sampling
|
|
85
85
|
Requires-Dist: netmiko>=4.0; extra == "sampling"
|
|
86
86
|
Provides-Extra: dev
|
|
@@ -161,6 +161,19 @@ each need their own SDK, and `gemini`/`vertexai` share `google-genai` — and
|
|
|
161
161
|
never silently skip) — add the specific `,<provider>` extra explicitly only
|
|
162
162
|
if installing outside of `dev`.
|
|
163
163
|
|
|
164
|
+
For tooling that expects plain `requirements.txt` files instead of pip
|
|
165
|
+
extras (Docker layers, offline pins, etc), `requirements/` has one
|
|
166
|
+
`requirements-<provider>.txt` per provider whose SDK isn't already
|
|
167
|
+
pinned by a shared package (`anthropic`/`openai`/`azure`/`bedrock`/`oci`/
|
|
168
|
+
`cohere`/`mistral`/`gemini`/`vertexai`) — each mirrors the matching
|
|
169
|
+
`pyproject.toml` extra exactly (`-e .` plus that provider's SDK pin), so
|
|
170
|
+
`pip install -r requirements/requirements-oci.txt` is equivalent to
|
|
171
|
+
`pip install -e ".[oci]"`. To also run that provider's tests, use the
|
|
172
|
+
matching `dev-<provider>.txt` instead — it layers `pytest`/`pytest-cov`
|
|
173
|
+
on top via `-r requirements-<provider>.txt`, so cloning the repo and
|
|
174
|
+
running `pip install -r requirements/dev-oci.txt` is enough on its own,
|
|
175
|
+
no separate install step needed.
|
|
176
|
+
|
|
164
177
|
Linting/formatting/type-checking/docs run through tox instead of extras — see
|
|
165
178
|
`tox.ini` (`tox -e lint`/`format`/`typecheck`/`docs`), each installing its own
|
|
166
179
|
tools in an isolated env. Cutting a release needs `pip install -e ".[release]"`
|
|
@@ -333,6 +346,38 @@ evaluating any gate — so running `promotion` alone after a `trial` run is enou
|
|
|
333
346
|
pick up new evidence; a separate `integration` run is only useful if you want to
|
|
334
347
|
inspect `reference-summary.json` without also promoting.
|
|
335
348
|
|
|
349
|
+
## Python API
|
|
350
|
+
|
|
351
|
+
Everything the CLI does is also a plain Python call — `parseforge/api.py` is the
|
|
352
|
+
single supported place to import from (also re-exported at the package root):
|
|
353
|
+
|
|
354
|
+
```python
|
|
355
|
+
from parseforge import CliContext, LLMProviderConfig, run_command_pipeline
|
|
356
|
+
from parseforge.naming import AnthropicRegexBuilder
|
|
357
|
+
from parseforge.sampling.backends import NetmikoSampler
|
|
358
|
+
from parseforge.sampling import DeviceConnection
|
|
359
|
+
|
|
360
|
+
result = run_command_pipeline(
|
|
361
|
+
"show clock",
|
|
362
|
+
CliContext(vendor="cisco", family="catalyst9200", os="ios-xe", version="17.9.1"),
|
|
363
|
+
DeviceConnection(host="10.0.0.1", username="admin", password="secret", device_type="cisco_ios"),
|
|
364
|
+
AnthropicRegexBuilder(api_key="sk-..."),
|
|
365
|
+
NetmikoSampler(),
|
|
366
|
+
LLMProviderConfig(provider="anthropic", api_key="sk-...", model="claude-haiku-4-5-20251001"),
|
|
367
|
+
)
|
|
368
|
+
print(result.cli_name, result.passed, result.total_usage)
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
One function per pipeline stage (SPEC.md §5): `cli_name`/`resolve_cli_name` (naming),
|
|
372
|
+
`sample` (sampling), `generate` (generation), `parse` (self-validation),
|
|
373
|
+
`run_command_pipeline` (steps 1-7 in one call), `build_integration`/
|
|
374
|
+
`write_reference_summary` (integration), `promote_auto`/`promote_user_reviewed`
|
|
375
|
+
(promotion), `check_drift` (drift monitoring) — plus the dataclasses/enums each one
|
|
376
|
+
returns or accepts. Provider-specific naming builders (`AnthropicRegexBuilder`,
|
|
377
|
+
`OCIRegexBuilder`, ...) aren't re-exported at the root — import them from
|
|
378
|
+
`parseforge.naming` directly. Anything not in `parseforge.api.__all__` is internal
|
|
379
|
+
and may change without notice.
|
|
380
|
+
|
|
336
381
|
## Reference
|
|
337
382
|
|
|
338
383
|
- [Documentation site](https://geeks-trident-llc.github.io/parseforge/) ([source](./docs/index.md))
|
|
@@ -57,6 +57,19 @@ each need their own SDK, and `gemini`/`vertexai` share `google-genai` — and
|
|
|
57
57
|
never silently skip) — add the specific `,<provider>` extra explicitly only
|
|
58
58
|
if installing outside of `dev`.
|
|
59
59
|
|
|
60
|
+
For tooling that expects plain `requirements.txt` files instead of pip
|
|
61
|
+
extras (Docker layers, offline pins, etc), `requirements/` has one
|
|
62
|
+
`requirements-<provider>.txt` per provider whose SDK isn't already
|
|
63
|
+
pinned by a shared package (`anthropic`/`openai`/`azure`/`bedrock`/`oci`/
|
|
64
|
+
`cohere`/`mistral`/`gemini`/`vertexai`) — each mirrors the matching
|
|
65
|
+
`pyproject.toml` extra exactly (`-e .` plus that provider's SDK pin), so
|
|
66
|
+
`pip install -r requirements/requirements-oci.txt` is equivalent to
|
|
67
|
+
`pip install -e ".[oci]"`. To also run that provider's tests, use the
|
|
68
|
+
matching `dev-<provider>.txt` instead — it layers `pytest`/`pytest-cov`
|
|
69
|
+
on top via `-r requirements-<provider>.txt`, so cloning the repo and
|
|
70
|
+
running `pip install -r requirements/dev-oci.txt` is enough on its own,
|
|
71
|
+
no separate install step needed.
|
|
72
|
+
|
|
60
73
|
Linting/formatting/type-checking/docs run through tox instead of extras — see
|
|
61
74
|
`tox.ini` (`tox -e lint`/`format`/`typecheck`/`docs`), each installing its own
|
|
62
75
|
tools in an isolated env. Cutting a release needs `pip install -e ".[release]"`
|
|
@@ -229,6 +242,38 @@ evaluating any gate — so running `promotion` alone after a `trial` run is enou
|
|
|
229
242
|
pick up new evidence; a separate `integration` run is only useful if you want to
|
|
230
243
|
inspect `reference-summary.json` without also promoting.
|
|
231
244
|
|
|
245
|
+
## Python API
|
|
246
|
+
|
|
247
|
+
Everything the CLI does is also a plain Python call — `parseforge/api.py` is the
|
|
248
|
+
single supported place to import from (also re-exported at the package root):
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
from parseforge import CliContext, LLMProviderConfig, run_command_pipeline
|
|
252
|
+
from parseforge.naming import AnthropicRegexBuilder
|
|
253
|
+
from parseforge.sampling.backends import NetmikoSampler
|
|
254
|
+
from parseforge.sampling import DeviceConnection
|
|
255
|
+
|
|
256
|
+
result = run_command_pipeline(
|
|
257
|
+
"show clock",
|
|
258
|
+
CliContext(vendor="cisco", family="catalyst9200", os="ios-xe", version="17.9.1"),
|
|
259
|
+
DeviceConnection(host="10.0.0.1", username="admin", password="secret", device_type="cisco_ios"),
|
|
260
|
+
AnthropicRegexBuilder(api_key="sk-..."),
|
|
261
|
+
NetmikoSampler(),
|
|
262
|
+
LLMProviderConfig(provider="anthropic", api_key="sk-...", model="claude-haiku-4-5-20251001"),
|
|
263
|
+
)
|
|
264
|
+
print(result.cli_name, result.passed, result.total_usage)
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
One function per pipeline stage (SPEC.md §5): `cli_name`/`resolve_cli_name` (naming),
|
|
268
|
+
`sample` (sampling), `generate` (generation), `parse` (self-validation),
|
|
269
|
+
`run_command_pipeline` (steps 1-7 in one call), `build_integration`/
|
|
270
|
+
`write_reference_summary` (integration), `promote_auto`/`promote_user_reviewed`
|
|
271
|
+
(promotion), `check_drift` (drift monitoring) — plus the dataclasses/enums each one
|
|
272
|
+
returns or accepts. Provider-specific naming builders (`AnthropicRegexBuilder`,
|
|
273
|
+
`OCIRegexBuilder`, ...) aren't re-exported at the root — import them from
|
|
274
|
+
`parseforge.naming` directly. Anything not in `parseforge.api.__all__` is internal
|
|
275
|
+
and may change without notice.
|
|
276
|
+
|
|
232
277
|
## Reference
|
|
233
278
|
|
|
234
279
|
- [Documentation site](https://geeks-trident-llc.github.io/parseforge/) ([source](./docs/index.md))
|
|
@@ -79,7 +79,7 @@ Keep the timestamp+shortid directories (not `result1..N`) — chronological orde
|
|
|
79
79
|
| `derive/template.textfsm` | Cleaned, DSL-compiled TextFSM template candidate (`.textfsm` extension — recognized by TextFSM tooling/linters, unlike `textfsm.template`) |
|
|
80
80
|
| `derive/readable-dsl.txt` | Human-readable description of what the template captures |
|
|
81
81
|
| `derive/recognizers.txt` | Heuristics/signatures for detecting this output type at runtime |
|
|
82
|
-
| `summary.json` | Everything else about the run: `created_at`/`ended_at`/`duration_ms`, `passed`, `error` (when `passed` is false), `metadata` (project/username/email/description), `command_info` (vendor/family/os/version/device_type/command), `usage` (naming + generation token counts
|
|
82
|
+
| `summary.json` | Everything else about the run: `created_at`/`ended_at`/`duration_ms`, `passed`, `error` (when `passed` is false), `metadata` (project/username/email/description), `command_info` (vendor/family/os/version/device_type/command), `usage` (naming + generation token counts), and `provider_info` (the generation provider/model) |
|
|
83
83
|
|
|
84
84
|
### 3.2 `integration/` (no human review yet)
|
|
85
85
|
|
|
@@ -180,4 +180,4 @@ Where a fourth tier is tempting but better handled as **metadata instead of a ne
|
|
|
180
180
|
- Do you want a **registry/index file** (e.g. `catalog.json`) at the repo root listing every `<vendor>/<family>/<os>/<cli-name>` combination that exists, plus its authoritative status, for fast lookup without walking the filesystem?
|
|
181
181
|
- What **confidence threshold** (match-rate %, sample count minimum) should gate auto-promotion vs. human review in step 9 — worth making this configurable per-project rather than hardcoded?
|
|
182
182
|
- Should `recognizers.txt` support **one-of-many matching** from day one (per the multi-variant note in §6), or is that a v2 concern?
|
|
183
|
-
- What's the **LLM provider/model** for generation — worth pinning per-project so `summary.json`'s usage
|
|
183
|
+
- What's the **LLM provider/model** for generation — worth pinning per-project so `summary.json`'s token usage is comparable across runs?
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
2
|
+
version = __version__
|
|
3
|
+
|
|
4
|
+
from .api import ( # noqa: E402
|
|
5
|
+
DEFAULT_STORE_ROOT,
|
|
6
|
+
CliContext,
|
|
7
|
+
DeviceConnection,
|
|
8
|
+
DeviceKey,
|
|
9
|
+
DriftCheckResult,
|
|
10
|
+
DriftGate,
|
|
11
|
+
GenerationResult,
|
|
12
|
+
GenerationTokenUsage,
|
|
13
|
+
GroupEvaluation,
|
|
14
|
+
LLMCLIResponse,
|
|
15
|
+
LLMProviderConfig,
|
|
16
|
+
Mode,
|
|
17
|
+
NamingResolution,
|
|
18
|
+
NamingTokenUsage,
|
|
19
|
+
ParseResult,
|
|
20
|
+
PromotionDecision,
|
|
21
|
+
PromotionGate,
|
|
22
|
+
PromotionMetadata,
|
|
23
|
+
PromotionMode,
|
|
24
|
+
PromotionRunResult,
|
|
25
|
+
Reference,
|
|
26
|
+
ReferenceGroup,
|
|
27
|
+
ReferenceVariant,
|
|
28
|
+
RegexBuilder,
|
|
29
|
+
Sampler,
|
|
30
|
+
TrialMetadata,
|
|
31
|
+
TrialResult,
|
|
32
|
+
UserReviewedRequest,
|
|
33
|
+
build_integration,
|
|
34
|
+
build_reference_summary,
|
|
35
|
+
check_drift,
|
|
36
|
+
cli_name,
|
|
37
|
+
decide_promotion,
|
|
38
|
+
discover_device_keys,
|
|
39
|
+
evaluate_cases,
|
|
40
|
+
generate,
|
|
41
|
+
parse,
|
|
42
|
+
promote_auto,
|
|
43
|
+
promote_user_reviewed,
|
|
44
|
+
resolve_cli_name,
|
|
45
|
+
run_command_pipeline,
|
|
46
|
+
sample,
|
|
47
|
+
write_reference_summary,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"version",
|
|
52
|
+
"__version__",
|
|
53
|
+
"cli_name",
|
|
54
|
+
"resolve_cli_name",
|
|
55
|
+
"NamingResolution",
|
|
56
|
+
"CliContext",
|
|
57
|
+
"RegexBuilder",
|
|
58
|
+
"LLMCLIResponse",
|
|
59
|
+
"NamingTokenUsage",
|
|
60
|
+
"sample",
|
|
61
|
+
"DeviceConnection",
|
|
62
|
+
"Sampler",
|
|
63
|
+
"generate",
|
|
64
|
+
"GenerationResult",
|
|
65
|
+
"GenerationTokenUsage",
|
|
66
|
+
"parse",
|
|
67
|
+
"ParseResult",
|
|
68
|
+
"run_command_pipeline",
|
|
69
|
+
"LLMProviderConfig",
|
|
70
|
+
"TrialMetadata",
|
|
71
|
+
"TrialResult",
|
|
72
|
+
"Mode",
|
|
73
|
+
"build_integration",
|
|
74
|
+
"build_reference_summary",
|
|
75
|
+
"write_reference_summary",
|
|
76
|
+
"Reference",
|
|
77
|
+
"ReferenceGroup",
|
|
78
|
+
"ReferenceVariant",
|
|
79
|
+
"promote_auto",
|
|
80
|
+
"promote_user_reviewed",
|
|
81
|
+
"decide_promotion",
|
|
82
|
+
"evaluate_cases",
|
|
83
|
+
"PromotionGate",
|
|
84
|
+
"PromotionMetadata",
|
|
85
|
+
"UserReviewedRequest",
|
|
86
|
+
"GroupEvaluation",
|
|
87
|
+
"PromotionRunResult",
|
|
88
|
+
"PromotionDecision",
|
|
89
|
+
"PromotionMode",
|
|
90
|
+
"check_drift",
|
|
91
|
+
"DriftGate",
|
|
92
|
+
"DriftCheckResult",
|
|
93
|
+
"DEFAULT_STORE_ROOT",
|
|
94
|
+
"DeviceKey",
|
|
95
|
+
"discover_device_keys",
|
|
96
|
+
]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Public Python API — the single place to import parseforge's stable,
|
|
2
|
+
supported surface from. Mirrors textfsm-ai's own api.py convention
|
|
3
|
+
(https://github.com/Geeks-Trident-LLC/textfsm-ai): one entry point per
|
|
4
|
+
pipeline stage (SPEC.md §5), plus the types each one returns or accepts.
|
|
5
|
+
|
|
6
|
+
Provider-specific naming builders (``AnthropicRegexBuilder``,
|
|
7
|
+
``OCIRegexBuilder``, ...) aren't re-exported here — they're already a
|
|
8
|
+
clean, documented import path via :mod:`parseforge.naming` directly, and
|
|
9
|
+
listing all eighteen here would bloat this module without adding
|
|
10
|
+
anything. Anything not listed in ``__all__`` (module-internal helpers,
|
|
11
|
+
provider implementation details, CLI plumbing) isn't part of the public
|
|
12
|
+
API and may change without notice.
|
|
13
|
+
|
|
14
|
+
``naming``'s and ``generation``'s ``TokenUsage`` classes are separate
|
|
15
|
+
types that happen to share a name (naming makes one call per cache
|
|
16
|
+
miss; generation's is already accumulated across every LLM call in its
|
|
17
|
+
own pipeline, see ``GenerationResult.usage``) — aliased here as
|
|
18
|
+
``NamingTokenUsage``/``GenerationTokenUsage`` to avoid the collision,
|
|
19
|
+
the same convention this package's own test suite already uses.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from .drift import DriftCheckResult, DriftGate, check_drift
|
|
25
|
+
from .generation import GenerationResult, generate
|
|
26
|
+
from .generation import TokenUsage as GenerationTokenUsage
|
|
27
|
+
from .integration import (
|
|
28
|
+
Reference,
|
|
29
|
+
ReferenceGroup,
|
|
30
|
+
ReferenceVariant,
|
|
31
|
+
build_integration,
|
|
32
|
+
build_reference_summary,
|
|
33
|
+
write_reference_summary,
|
|
34
|
+
)
|
|
35
|
+
from .naming import (
|
|
36
|
+
CliContext,
|
|
37
|
+
LLMCLIResponse,
|
|
38
|
+
NamingResolution,
|
|
39
|
+
RegexBuilder,
|
|
40
|
+
cli_name,
|
|
41
|
+
resolve_cli_name,
|
|
42
|
+
)
|
|
43
|
+
from .naming import TokenUsage as NamingTokenUsage
|
|
44
|
+
from .paths import DEFAULT_STORE_ROOT, DeviceKey, discover_device_keys
|
|
45
|
+
from .pipeline import (
|
|
46
|
+
LLMProviderConfig,
|
|
47
|
+
Mode,
|
|
48
|
+
TrialMetadata,
|
|
49
|
+
TrialResult,
|
|
50
|
+
run_command_pipeline,
|
|
51
|
+
)
|
|
52
|
+
from .promotion import (
|
|
53
|
+
GroupEvaluation,
|
|
54
|
+
PromotionDecision,
|
|
55
|
+
PromotionGate,
|
|
56
|
+
PromotionMetadata,
|
|
57
|
+
PromotionMode,
|
|
58
|
+
PromotionRunResult,
|
|
59
|
+
UserReviewedRequest,
|
|
60
|
+
decide_promotion,
|
|
61
|
+
evaluate_cases,
|
|
62
|
+
promote_auto,
|
|
63
|
+
promote_user_reviewed,
|
|
64
|
+
)
|
|
65
|
+
from .sampling import DeviceConnection, Sampler, sample
|
|
66
|
+
from .validation import ParseResult, parse
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
# Naming (SPEC.md §2) — resolve a raw CLI command to its canonical
|
|
70
|
+
# cli-name, cached after the first LLM call.
|
|
71
|
+
"cli_name",
|
|
72
|
+
"resolve_cli_name",
|
|
73
|
+
"NamingResolution",
|
|
74
|
+
"CliContext",
|
|
75
|
+
"RegexBuilder",
|
|
76
|
+
"LLMCLIResponse",
|
|
77
|
+
"NamingTokenUsage",
|
|
78
|
+
# Sampling (SPEC.md §3) — capture raw command output from a device.
|
|
79
|
+
"sample",
|
|
80
|
+
"DeviceConnection",
|
|
81
|
+
"Sampler",
|
|
82
|
+
# Generation (SPEC.md §5 steps 5-6) — sample -> candidate TextFSM
|
|
83
|
+
# template, via textfsm-ai's delivery pipeline.
|
|
84
|
+
"generate",
|
|
85
|
+
"GenerationResult",
|
|
86
|
+
"GenerationTokenUsage",
|
|
87
|
+
# Validation (SPEC.md §5 step 7) — self-validate a template against
|
|
88
|
+
# its own sample.
|
|
89
|
+
"parse",
|
|
90
|
+
"ParseResult",
|
|
91
|
+
# Pipeline orchestration (SPEC.md §5) — one call runs steps 1-7
|
|
92
|
+
# (naming -> sampling -> generation -> validation) for a single trial.
|
|
93
|
+
"run_command_pipeline",
|
|
94
|
+
"LLMProviderConfig",
|
|
95
|
+
"TrialMetadata",
|
|
96
|
+
"TrialResult",
|
|
97
|
+
"Mode",
|
|
98
|
+
# Integration (SPEC.md §5 step 8) — cluster trials by output schema.
|
|
99
|
+
"build_integration",
|
|
100
|
+
"build_reference_summary",
|
|
101
|
+
"write_reference_summary",
|
|
102
|
+
"Reference",
|
|
103
|
+
"ReferenceGroup",
|
|
104
|
+
"ReferenceVariant",
|
|
105
|
+
# Promotion (SPEC.md §5 step 9) — auto-promote groups that clear
|
|
106
|
+
# their gate; queue everything else for human review.
|
|
107
|
+
"promote_auto",
|
|
108
|
+
"promote_user_reviewed",
|
|
109
|
+
"decide_promotion",
|
|
110
|
+
"evaluate_cases",
|
|
111
|
+
"PromotionGate",
|
|
112
|
+
"PromotionMetadata",
|
|
113
|
+
"UserReviewedRequest",
|
|
114
|
+
"GroupEvaluation",
|
|
115
|
+
"PromotionRunResult",
|
|
116
|
+
"PromotionDecision",
|
|
117
|
+
"PromotionMode",
|
|
118
|
+
# Drift monitoring (SPEC.md §5 step 11) — check an authoritative
|
|
119
|
+
# template against new production samples.
|
|
120
|
+
"check_drift",
|
|
121
|
+
"DriftGate",
|
|
122
|
+
"DriftCheckResult",
|
|
123
|
+
# Store-root layout helpers.
|
|
124
|
+
"DEFAULT_STORE_ROOT",
|
|
125
|
+
"DeviceKey",
|
|
126
|
+
"discover_device_keys",
|
|
127
|
+
]
|
|
@@ -659,6 +659,7 @@ def run_cmd(
|
|
|
659
659
|
click.echo(f"cli_name : {result.cli_name}")
|
|
660
660
|
click.echo(f"passed : {result.passed}")
|
|
661
661
|
click.echo(f"run_dir : {result.run_dir}")
|
|
662
|
+
click.echo(f"usage : {result.total_usage}")
|
|
662
663
|
|
|
663
664
|
|
|
664
665
|
def _check_connector(
|
|
@@ -1200,7 +1201,7 @@ def trial_cmd(config_path: str, store_root_opt: str | None) -> None:
|
|
|
1200
1201
|
for command, result in zip(cfg.commands, results):
|
|
1201
1202
|
click.echo(
|
|
1202
1203
|
f"{command} -> cli_name={result.cli_name} passed={result.passed} "
|
|
1203
|
-
f"run_dir={result.run_dir}"
|
|
1204
|
+
f"run_dir={result.run_dir} usage={result.total_usage}"
|
|
1204
1205
|
)
|
|
1205
1206
|
passed_count += int(result.passed)
|
|
1206
1207
|
click.echo(f"{passed_count}/{len(results)} passed")
|
|
@@ -44,7 +44,6 @@ class TokenUsage:
|
|
|
44
44
|
input_tokens: int
|
|
45
45
|
output_tokens: int
|
|
46
46
|
total_tokens: int
|
|
47
|
-
estimated_cost: float
|
|
48
47
|
|
|
49
48
|
|
|
50
49
|
@dataclass(frozen=True)
|
|
@@ -95,7 +94,6 @@ def generate(
|
|
|
95
94
|
input_tokens=usage_data.get("input_tokens", 0),
|
|
96
95
|
output_tokens=usage_data.get("output_tokens", 0),
|
|
97
96
|
total_tokens=usage_data.get("total_tokens", 0),
|
|
98
|
-
estimated_cost=usage_data.get("estimated_cost", 0.0),
|
|
99
97
|
)
|
|
100
98
|
|
|
101
99
|
gen_stage = (debug.get("generation_pipeline") or {}).get("last_stage") or {}
|
|
@@ -6,7 +6,6 @@ import time
|
|
|
6
6
|
from typing import TYPE_CHECKING, Any
|
|
7
7
|
|
|
8
8
|
from ..llm import CliContext, LLMCLIResponse, TokenUsage, build_prompt
|
|
9
|
-
from .cost import estimate_cost
|
|
10
9
|
from .errors import format_llm_error_reason, is_retryable
|
|
11
10
|
from .models import default_model
|
|
12
11
|
from .text import extract_pattern
|
|
@@ -89,9 +88,7 @@ class AnthropicRegexBuilder:
|
|
|
89
88
|
return LLMCLIResponse(
|
|
90
89
|
content="",
|
|
91
90
|
raw=exc,
|
|
92
|
-
usage=TokenUsage(
|
|
93
|
-
input_tokens=0, output_tokens=0, total_tokens=0, estimated_cost=0.0
|
|
94
|
-
),
|
|
91
|
+
usage=TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
|
95
92
|
duration_ms=(time.monotonic() - start) * 1000,
|
|
96
93
|
reason=format_llm_error_reason(exc),
|
|
97
94
|
ready=False,
|
|
@@ -111,13 +108,6 @@ class AnthropicRegexBuilder:
|
|
|
111
108
|
input_tokens=response.usage.input_tokens,
|
|
112
109
|
output_tokens=response.usage.output_tokens,
|
|
113
110
|
total_tokens=total_tokens,
|
|
114
|
-
estimated_cost=estimate_cost(
|
|
115
|
-
input_tokens=response.usage.input_tokens,
|
|
116
|
-
output_tokens=response.usage.output_tokens,
|
|
117
|
-
total_tokens=total_tokens,
|
|
118
|
-
provider=self.provider,
|
|
119
|
-
model=self.model,
|
|
120
|
-
),
|
|
121
111
|
),
|
|
122
112
|
duration_ms=duration_ms,
|
|
123
113
|
reason=response.stop_reason or "",
|
|
@@ -17,7 +17,6 @@ import time
|
|
|
17
17
|
from typing import TYPE_CHECKING, Any
|
|
18
18
|
|
|
19
19
|
from ..llm import CliContext, LLMCLIResponse, TokenUsage, build_prompt
|
|
20
|
-
from .cost import estimate_cost
|
|
21
20
|
from .text import extract_pattern
|
|
22
21
|
|
|
23
22
|
if TYPE_CHECKING:
|
|
@@ -161,9 +160,7 @@ class AzureRegexBuilder:
|
|
|
161
160
|
return LLMCLIResponse(
|
|
162
161
|
content="",
|
|
163
162
|
raw=exc,
|
|
164
|
-
usage=TokenUsage(
|
|
165
|
-
input_tokens=0, output_tokens=0, total_tokens=0, estimated_cost=0.0
|
|
166
|
-
),
|
|
163
|
+
usage=TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
|
167
164
|
duration_ms=(time.monotonic() - start) * 1000,
|
|
168
165
|
reason=_format_error_reason(exc),
|
|
169
166
|
ready=False,
|
|
@@ -178,18 +175,6 @@ class AzureRegexBuilder:
|
|
|
178
175
|
input_tokens=response.usage.prompt_tokens,
|
|
179
176
|
output_tokens=response.usage.completion_tokens,
|
|
180
177
|
total_tokens=response.usage.total_tokens,
|
|
181
|
-
estimated_cost=estimate_cost(
|
|
182
|
-
input_tokens=response.usage.prompt_tokens,
|
|
183
|
-
output_tokens=response.usage.completion_tokens,
|
|
184
|
-
total_tokens=response.usage.total_tokens,
|
|
185
|
-
provider=self.provider,
|
|
186
|
-
# No public pricing table for a customer's own Azure
|
|
187
|
-
# deployment — estimate_cost() falls back to 0.0 with
|
|
188
|
-
# a warning for an unrecognized (provider, model) pair,
|
|
189
|
-
# which is the correct/expected outcome here, not an
|
|
190
|
-
# error condition.
|
|
191
|
-
model=self._resolved_deployment or "",
|
|
192
|
-
),
|
|
193
178
|
),
|
|
194
179
|
duration_ms=duration_ms,
|
|
195
180
|
reason=choice.finish_reason or "",
|
|
@@ -17,7 +17,6 @@ import time
|
|
|
17
17
|
from typing import TYPE_CHECKING, Any
|
|
18
18
|
|
|
19
19
|
from ..llm import CliContext, LLMCLIResponse, TokenUsage, build_prompt
|
|
20
|
-
from .cost import estimate_cost
|
|
21
20
|
from .models import default_model
|
|
22
21
|
from .text import extract_pattern
|
|
23
22
|
|
|
@@ -138,9 +137,7 @@ class BedrockRegexBuilder:
|
|
|
138
137
|
return LLMCLIResponse(
|
|
139
138
|
content="",
|
|
140
139
|
raw=exc,
|
|
141
|
-
usage=TokenUsage(
|
|
142
|
-
input_tokens=0, output_tokens=0, total_tokens=0, estimated_cost=0.0
|
|
143
|
-
),
|
|
140
|
+
usage=TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
|
144
141
|
duration_ms=(time.monotonic() - start) * 1000,
|
|
145
142
|
reason=_format_error_reason(exc),
|
|
146
143
|
ready=False,
|
|
@@ -162,13 +159,6 @@ class BedrockRegexBuilder:
|
|
|
162
159
|
input_tokens=input_tokens,
|
|
163
160
|
output_tokens=output_tokens,
|
|
164
161
|
total_tokens=total_tokens,
|
|
165
|
-
estimated_cost=estimate_cost(
|
|
166
|
-
input_tokens=input_tokens,
|
|
167
|
-
output_tokens=output_tokens,
|
|
168
|
-
total_tokens=total_tokens,
|
|
169
|
-
provider=self.provider,
|
|
170
|
-
model=self.model,
|
|
171
|
-
),
|
|
172
162
|
),
|
|
173
163
|
duration_ms=duration_ms,
|
|
174
164
|
reason=stop_reason,
|
|
@@ -12,7 +12,6 @@ import time
|
|
|
12
12
|
from typing import TYPE_CHECKING, Any
|
|
13
13
|
|
|
14
14
|
from ..llm import CliContext, LLMCLIResponse, TokenUsage, build_prompt
|
|
15
|
-
from .cost import estimate_cost
|
|
16
15
|
from .errors import format_llm_error_reason, is_retryable
|
|
17
16
|
from .models import default_model
|
|
18
17
|
from .text import extract_pattern
|
|
@@ -97,9 +96,7 @@ class CerebrasRegexBuilder:
|
|
|
97
96
|
return LLMCLIResponse(
|
|
98
97
|
content="",
|
|
99
98
|
raw=exc,
|
|
100
|
-
usage=TokenUsage(
|
|
101
|
-
input_tokens=0, output_tokens=0, total_tokens=0, estimated_cost=0.0
|
|
102
|
-
),
|
|
99
|
+
usage=TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
|
103
100
|
duration_ms=(time.monotonic() - start) * 1000,
|
|
104
101
|
reason=format_llm_error_reason(exc),
|
|
105
102
|
ready=False,
|
|
@@ -114,13 +111,6 @@ class CerebrasRegexBuilder:
|
|
|
114
111
|
input_tokens=response.usage.prompt_tokens,
|
|
115
112
|
output_tokens=response.usage.completion_tokens,
|
|
116
113
|
total_tokens=response.usage.total_tokens,
|
|
117
|
-
estimated_cost=estimate_cost(
|
|
118
|
-
input_tokens=response.usage.prompt_tokens,
|
|
119
|
-
output_tokens=response.usage.completion_tokens,
|
|
120
|
-
total_tokens=response.usage.total_tokens,
|
|
121
|
-
provider=self.provider,
|
|
122
|
-
model=self.model,
|
|
123
|
-
),
|
|
124
114
|
),
|
|
125
115
|
duration_ms=duration_ms,
|
|
126
116
|
reason=choice.finish_reason or "",
|
|
@@ -15,7 +15,6 @@ import time
|
|
|
15
15
|
from typing import TYPE_CHECKING, Any, cast
|
|
16
16
|
|
|
17
17
|
from ..llm import CliContext, LLMCLIResponse, TokenUsage, build_prompt
|
|
18
|
-
from .cost import estimate_cost
|
|
19
18
|
from .models import default_model
|
|
20
19
|
from .text import extract_pattern
|
|
21
20
|
|
|
@@ -133,9 +132,7 @@ class CohereRegexBuilder:
|
|
|
133
132
|
return LLMCLIResponse(
|
|
134
133
|
content="",
|
|
135
134
|
raw=exc,
|
|
136
|
-
usage=TokenUsage(
|
|
137
|
-
input_tokens=0, output_tokens=0, total_tokens=0, estimated_cost=0.0
|
|
138
|
-
),
|
|
135
|
+
usage=TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0),
|
|
139
136
|
duration_ms=(time.monotonic() - start) * 1000,
|
|
140
137
|
reason=_format_error_reason(exc),
|
|
141
138
|
ready=False,
|
|
@@ -159,13 +156,6 @@ class CohereRegexBuilder:
|
|
|
159
156
|
input_tokens=input_tokens,
|
|
160
157
|
output_tokens=output_tokens,
|
|
161
158
|
total_tokens=total_tokens,
|
|
162
|
-
estimated_cost=estimate_cost(
|
|
163
|
-
input_tokens=input_tokens,
|
|
164
|
-
output_tokens=output_tokens,
|
|
165
|
-
total_tokens=total_tokens,
|
|
166
|
-
provider=self.provider,
|
|
167
|
-
model=self.model,
|
|
168
|
-
),
|
|
169
159
|
),
|
|
170
160
|
duration_ms=duration_ms,
|
|
171
161
|
reason=response.finish_reason or "",
|