parseforge 0.2.1__tar.gz

parseforge-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Geeks Trident LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

parseforge-0.2.1/MANIFEST.in

@@ -0,0 +1,12 @@
+include LICENSE
+include README.md
+include SPEC.md
+
+# Include any YAML/JSON data files inside the package (future-proof)
+recursive-include parseforge *.yaml
+recursive-include parseforge *.json
+
+# Exclude tests, docs, generated template store from source distribution
+prune tests
+prune docs
+prune store

parseforge-0.2.1/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: parseforge
+Version: 0.2.1
+Summary: LLM-driven pipeline that forges, validates, and promotes TextFSM templates from network CLI output
+Author-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
+Maintainer-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
+License: MIT
+Keywords: textfsm,network automation,cli parsing,template generator,ai,llm,netmiko,device output parsing
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: textfsm>=1.1.0
+Provides-Extra: sampling
+Requires-Dist: netmiko>=4.0; extra == "sampling"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: mypy>=1.10.0; extra == "dev"
+Dynamic: license-file
+
+# ParseForge
+
+LLM-driven pipeline that forges, cross-validates, and promotes [TextFSM](https://github.com/google/textfsm)
+templates from network device CLI output.
+
+Full design plan: [SPEC.md](SPEC.md).
+
+## Layout
+
+```
+parseforge/
+  naming.py       CLI command -> canonical cli-name (SPEC §2)
+  paths.py        trials/integration/authoritative path resolution (SPEC §3)
+  sampling.py     device connection + command capture (SPEC §5 step 4)
+  generation.py   LLM call + template extraction (SPEC §5 steps 5-6)
+  validation.py   self-validation + cross-validation scoring (SPEC §5 steps 7-8)
+  promotion.py    authoritative promotion gate + drift status (SPEC §3.3, §5 steps 9-10)
+  pipeline.py     orchestrates the above; Mode LOOP (MVP) vs Mode BATCH (SPEC §4)
+  cli/            `parseforge` command-line entry point
+
+store/            runtime template tree: trials/ -> integration/ -> authoritative/
+                  (generated at runtime, not checked in — see .gitignore)
+```
+
+`naming.py` and `paths.py` are implemented and tested. `sampling.py`, `generation.py`,
+`validation.py`, `promotion.py`, and `pipeline.py` are stub interfaces — the SPEC.md
+section cited in each docstring is the next thing to implement.
+
+## Development
+
+```
+pip install -e ".[dev,sampling]"
+pytest
+```
+
+## CLI
+
+```
+parseforge name show interface GE1.1 status
+# -> show-interface-var1-status
+```

parseforge-0.2.1/README.md

@@ -0,0 +1,41 @@
+# ParseForge
+
+LLM-driven pipeline that forges, cross-validates, and promotes [TextFSM](https://github.com/google/textfsm)
+templates from network device CLI output.
+
+Full design plan: [SPEC.md](SPEC.md).
+
+## Layout
+
+```
+parseforge/
+  naming.py       CLI command -> canonical cli-name (SPEC §2)
+  paths.py        trials/integration/authoritative path resolution (SPEC §3)
+  sampling.py     device connection + command capture (SPEC §5 step 4)
+  generation.py   LLM call + template extraction (SPEC §5 steps 5-6)
+  validation.py   self-validation + cross-validation scoring (SPEC §5 steps 7-8)
+  promotion.py    authoritative promotion gate + drift status (SPEC §3.3, §5 steps 9-10)
+  pipeline.py     orchestrates the above; Mode LOOP (MVP) vs Mode BATCH (SPEC §4)
+  cli/            `parseforge` command-line entry point
+
+store/            runtime template tree: trials/ -> integration/ -> authoritative/
+                  (generated at runtime, not checked in — see .gitignore)
+```
+
+`naming.py` and `paths.py` are implemented and tested. `sampling.py`, `generation.py`,
+`validation.py`, `promotion.py`, and `pipeline.py` are stub interfaces — the SPEC.md
+section cited in each docstring is the next thing to implement.
+
+## Development
+
+```
+pip install -e ".[dev,sampling]"
+pytest
+```
+
+## CLI
+
+```
+parseforge name show interface GE1.1 status
+# -> show-interface-var1-status
+```

parseforge-0.2.1/SPEC.md

@@ -0,0 +1,173 @@
+# Network CLI → TextFSM Template Generator — Project Plan & Requirements
+
+## 1. Project & Repo Naming
+
+| Candidate | Repo name (kebab-case) | Rationale |
+|---|---|---|
+| **ParseForge** | `parseforge` | Short, memorable, clearly about forging parsers. Available-sounding, no CLI/network jargon lock-in if you expand beyond Cisco later. |
+| **CLIFsm** | `cli-fsm-forge` | Literal — CLI + FSM (finite state machine, what TextFSM templates are). Good for discoverability if someone searches "textfsm" + "cli". |
+| **ShowParse AI** | `showparse-ai` | Leans into the fact that almost every input is a `show` command. Very descriptive, less generic. |
+| **NetFSM** | `netfsm` | Shortest option. Good if this becomes a pip-installable library/CLI tool people type often. |
+
+**Recommendation:** `parseforge` as the project name, `parseforge` (or `parseforge-textfsm`) as the repo name. It reads well as a CLI tool name (`parseforge run ...`), isn't locked to Cisco/TextFSM specifically if you extend to other vendors or output formats (e.g., ntc-templates-style YAML, TTP, or Genie parsers) later, and is easy to say out loud.
+
+If you want to keep `textfsm-ai` since that's the term you're already using internally/in docs, that's a perfectly reasonable literal alternative — just know it reads as "TextFSM, but AI" rather than as a product name, which is fine for an internal tool but less distinctive if this is ever open-sourced.
+
+---
+
+## 2. CLI-to-Name Translation
+
+**Rule:** Tokenize the full command string. Classify each token as **literal** (fixed across all invocations of this command type) or **variable** (an identifier — interface name, VLAN ID, IP, hostname, etc.).
+
+**Naming convention — always index variables, even when there's only one:**
+
+```
+show version                         → show-version
+show interface GE1.1 status          → show-interface-var1-status
+show interface GE1.1 counters        → show-interface-var1-counters
+show ip route 10.0.0.0                → show-ip-route-var1
+show bgp neighbors 10.1.1.1 advertised-routes
+                                       → show-bgp-neighbors-var1-advertised-routes
+```
+
+**Why always number, even for a single variable (`var1` not `var`):**
+A naming function that omits the index for single-variable commands has to *already know* the final variable count before naming — but that can change: a newer OS version might add an optional second argument to the same command family, or you might discover a command has two variable slots you didn't notice on first sample. Indexing consistently means:
+- The name-generation code is a pure, stateless function of token position — no lookahead needed.
+- No silent renames/migrations later if a command turns out to have more variables than expected.
+- Multi-variable commands (2+) stay unambiguous by construction — `var1`, `var2` in left-to-right order of appearance.
+
+This differs slightly from the older convention used in the `ntc-templates` community library (which just drops variable tokens entirely rather than placeholding them — e.g. `cisco_ios_show_interfaces.textfsm` regardless of interface name). That works for them because their catalog is hand-curated and collisions are manually resolved. Since your pipeline is LLM-driven and needs to run unattended, explicit indexed placeholders are safer and self-documenting.
+
+**Edge case — literal tokens that look variable-ish:** keywords like `brief`, `status`, `detail`, `counters`, `summary` are still literal (they're fixed sub-command keywords, not user-supplied values) and stay as literal tokens in the name.
+
+---
+
+## 3. Storage Layout — Three-Tier Promotion
+
+This mirrors the candidate → staging → production pattern used for ML model registries, applied to templates: **trials** (raw, unreviewed attempts) → **integration** (cross-validated candidates, pending approval) → **authoritative** (approved, in production use, drift-monitored).
+
+Shared path prefix under all three tiers:
+```
+<vendor>/<device-family>/<os>/<version>/<cli-name>/
+```
+e.g. `cisco/catalyst9200/ios-xe/17.9.1/show-interface-var1-status/`
+
+Note: use `ios-xe` / `nx-os` (hyphenated) rather than `xe` / `ios` alone once multiple Cisco OS families share the tree — `xe` alone is ambiguous without the `ios-` prefix once IOS classic, IOS XE, and IOS XR all live under `cisco/`.
+
+### 3.1 `trials/`
+
+```
+trials/<vendor>/<family>/<os>/<version>/<cli-name>/
+  <yyyymmdd-HHMMSS-shortid>/
+    input.txt
+    raw-llm-response.txt
+    usage.txt
+    raw-template
+    template.textfsm
+    readable-dsl.txt
+    recognizers.txt
+    llm-records.json
+    records.json
+    debug.txt
+    status.txt
+```
+Keep the timestamp+shortid directories (not `result1..N`) — chronological ordering and collision-safety in batch mode come for free, and sequential numbers throw both away.
+
+| File | Purpose |
+|---|---|
+| `input.txt` | Raw CLI output sample fed to the LLM |
+| `raw-llm-response.txt` | Full, unprocessed LLM response |
+| `usage.txt` | Token counts / cost metadata for that call |
+| `raw-template` | Template as extracted from the LLM response, pre-cleanup |
+| `template.textfsm` | Cleaned TextFSM template candidate (`.textfsm` extension — recognized by TextFSM tooling/linters, unlike `textfsm.template`) |
+| `readable-dsl.txt` | Human-readable description of what the template captures |
+| `recognizers.txt` | Heuristics/signatures for detecting this output type at runtime |
+| `llm-records.json` | Structured record of the LLM interaction (prompt, model, params, timestamps) |
+| `records.json` | Result of running this trial's `template.textfsm` against its own `input.txt` |
+| `debug.txt` | Parse errors, warnings, retry attempts |
+| `status.txt` | `passed`/`failed`, `llm-duration-ms`, `pipeline-duration-ms`, error summary |
+
+### 3.2 `integration/` (no human review yet)
+
+```
+integration/<vendor>/<family>/<os>/<version>/<cli-name>/
+  common-result/
+    template.textfsm       ← the winning candidate, copied from trials/
+    selection-report.json  ← which trials were considered, why this one won
+    artifact/               ← supporting evidence (see below)
+```
+
+**How "common" gets chosen — don't just self-validate:** test *every* candidate `template.textfsm` from `trials/<cli-name>/` against *every* `input.txt` collected for that `cli-name`, not only the sample it was generated from. Promote whichever template parses the largest share of all known samples correctly. A template that only works against its own source sample is exactly the overfit case integration should catch. If you generate multiple candidates from the same input (repeated LLM calls), a structure that recurs across generations is additional evidence of stability — worth folding into the score, not just accuracy alone.
+
+`selection-report.json` should record: which trial run IDs were candidates, the cross-validation match rate for each, which samples the winner failed on (if any), and the score that made it win. This is what a human reviewer actually reads — the raw template alone doesn't convey robustness.
+
+`artifact/` holds whatever a reviewer needs to judge the result without re-running the pipeline: the aggregated `records.json` outputs across all cross-validated samples, and a diff/summary of any samples where the winning template didn't fully match.
+
+### 3.3 `authoritative/` (approved via human review, or confidence-gated auto-promotion)
+
+```
+authoritative/<vendor>/<family>/<os>/<version>/<cli-name>/
+  template.textfsm          ← current approved template
+  artifact/                  ← carried over from the winning integration candidate
+  history/
+    <yyyymmdd-HHMMSS>-<shortid>.textfsm   ← every prior approved version, kept
+  promotion-log.json         ← who/what approved each version, when, why, and what it replaced
+  drift-log.json             ← match-rate over time against live production samples
+```
+
+Promotion from `integration/common-result` to `authoritative/` should default to human review, but can be confidence-gated: if the cross-validation match rate is at or near 100% across all known samples with no ambiguous fields, auto-promote; otherwise queue for human review. This isn't a replacement for review — it's a filter that keeps the easy, unambiguous cases from waiting on a person while still requiring a human on anything uncertain. A custom drift/quality-scoring model can slot into this same gate later without changing the pipeline shape.
+
+**Drift detection** runs the current `authoritative/.../template.textfsm` against new production samples on an ongoing basis, logging match rate to `drift-log.json`. When match rate drops below threshold, that failing sample is written into `trials/.../<cli-name>/` as a new run, which re-enters the pipeline at §5 step 4 and works its way back through `integration/` to a possible new `authoritative/` version — closing the loop rather than treating drift as a one-off alert.
+
+---
+
+## 4. Execution Modes
+
+**Mode 1 — Batch (gather-then-generate):**
+- Pro: LLM can be shown patterns across multiple samples of the *same* command at once (e.g., three different interface states) in one shot, likely improving template robustness on the first try.
+- Con: Slower feedback loop — you don't know command #3 has a connectivity problem until after collecting #1–#10.
+
+**Mode 2 — Loop (sample → generate, per command):**
+- Pro: Fails fast and isolates problems per-command; simpler to debug; easier to parallelize across commands.
+- Con: Each template is generated from a single sample unless you explicitly loop multiple times per command and merge.
+
+**Recommendation:** Build Mode 2 first as the MVP — it's the simpler pipeline and gives you per-command `status.txt` results immediately. Add Mode 1 as a config flag afterward that changes only the *sampling* stage (collect N samples per command before invoking the LLM) — the generation, storage, and validation stages stay identical between modes if you design sampling as a separable stage up front.
+
+---
+
+## 5. Pipeline Steps
+
+1. **Input intake** — device OS/version/family, auth, command list, mode selection.
+2. **Name generation** — tokenize each command → canonical `cli-name` per §2.
+3. **Path resolution** — compute `<vendor>/<device-family>/<os>/<version>/<cli-name>/` per §3.
+4. **Sampling** — connect (Netmiko/similar), run command(s), capture raw output → `trials/.../<run-id>/input.txt`.
+5. **Generation** — send `input.txt` (+ prior context if Mode 1) to LLM → `raw-llm-response.txt`, `usage.txt`.
+6. **Extraction & cleanup** — pull template from response → `raw-template` → cleaned `template.textfsm`.
+7. **Self-validation** — run `template.textfsm` against its own `input.txt` → `records.json`; capture errors → `debug.txt`; write trial `status.txt`.
+8. **Integration selection** — cross-validate every candidate `template.textfsm` in `trials/.../<cli-name>/` against every known `input.txt` for that `cli-name`; promote the best-scoring one to `integration/.../common-result/`, writing `selection-report.json`.
+9. **Authoritative promotion** — auto-promote if the winning candidate clears the confidence threshold across all known samples; otherwise queue for human review. On approval, copy to `authoritative/.../template.textfsm`, archive the prior version to `history/`, and write `promotion-log.json`.
+10. **Drift monitoring** — continuously run the authoritative template against new production samples, logging match rate to `drift-log.json`. On breach, feed the failing sample back into `trials/` (→ step 4) to generate a replacement candidate.
+11. **Repeat/loop or batch-complete** depending on mode.
+
+---
+
+## 6. Design Notes: Is Three-Tier Enough?
+
+**Yes, for the promotion pipeline itself.** `trials → integration → authoritative` covers the full lifecycle a template needs to go through to earn trust: generate, cross-validate, approve. Adding a fourth *promotion* tier (e.g. a separate "staging" between integration and authoritative) wouldn't buy you anything distinct — integration already *is* the staging step, since nothing in it is live yet.
+
+Where a fourth tier is tempting but better handled as **metadata instead of a new directory layer:**
+
+- **Quarantine / stale-but-still-serving:** when drift is detected, the authoritative template is still the one in production use — you don't want to yank it out mid-flight while a replacement works its way through the pipeline. Rather than a new top-level tier, this is a `status` field in `drift-log.json` (`ok` / `drifting` / `superseded`) on the existing authoritative entry. The template keeps serving; the status just flags that a replacement is in flight.
+- **Archived / deprecated (EOL device-OS combos):** for template families you no longer actively maintain (e.g. an OS version taken out of production fleet-wide), this is also a status flag rather than a directory move — `history/` already retains every prior version, so "archived" just means promotion-log stops getting new entries, not that files need to relocate.
+- **Multiple simultaneously-valid templates for one `cli-name`:** if a command's output legitimately varies by hardware config (e.g. a chassis with vs. without an optional module) rather than by drift, that's not a tier problem — it's a case for storing *multiple* authoritative templates under the same `cli-name` path with distinct `recognizers.txt` signatures, and letting runtime dispatch pick the right one. Worth flagging now so `recognizers.txt` is designed to support one-of-many matching from the start rather than assuming exactly one template per `cli-name`.
+
+**Bottom line:** keep the three directories; push the "in-between" states (drifting-but-live, archived, multi-variant) into metadata/status fields on the existing tiers. Adding more top-level directories per edge case leads to directory sprawl without adding real distinctions in *how* a template is used.
+
+---
+
+## 7. Open Questions for Next Iteration
+
+- Do you want a **registry/index file** (e.g. `catalog.json`) at the repo root listing every `<vendor>/<family>/<os>/<version>/<cli-name>` combination that exists, plus its authoritative status, for fast lookup without walking the filesystem?
+- What **confidence threshold** (match-rate %, sample count minimum) should gate auto-promotion vs. human review in step 9 — worth making this configurable per-project rather than hardcoded?
+- Should `recognizers.txt` support **one-of-many matching** from day one (per the multi-variant note in §6), or is that a v2 concern?
+- What's the **LLM provider/model** for generation — worth pinning per-project so `usage.txt` costs are comparable across runs?

parseforge-0.2.1/parseforge/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

parseforge-0.2.1/parseforge/cli/main.py

@@ -0,0 +1,55 @@
+"""parseforge CLI entry point."""
+
+from __future__ import annotations
+
+import click
+
+from parseforge import naming
+
+
+@click.group()
+@click.version_option(package_name="parseforge")
+def main() -> None:
+    """ParseForge — forge, validate, and promote TextFSM templates from CLI output."""
+
+
+@main.command("name")
+@click.option("--vendor", required=True)
+@click.option("--family", required=True)
+@click.option("--os", "os_", required=True)
+@click.option("--version", required=True)
+@click.argument("command", nargs=-1, required=True)
+def name_cmd(
+    vendor: str, family: str, os_: str, version: str, command: tuple[str, ...]
+) -> None:
+    """Print the canonical cli-name for a raw CLI COMMAND.
+
+    Looked up in the local cache first; only sent to an LLM (via a
+    RegexBuilder) on a cache miss. No RegexBuilder is wired into the
+    scaffold yet, so this only succeeds for commands already in
+    ~/.parseforge/.cli-name.json until one is implemented.
+    """
+    context = naming.CliContext(vendor=vendor, family=family, os=os_, version=version)
+    click.echo(naming.cli_name(" ".join(command), context))
+
+
+@main.command("run")
+@click.option("--vendor", required=True)
+@click.option("--family", required=True)
+@click.option("--os", "os_", required=True)
+@click.option("--version", required=True)
+@click.argument("command", nargs=-1, required=True)
+def run_cmd(
+    vendor: str, family: str, os_: str, version: str, command: tuple[str, ...]
+) -> None:
+    """Run the pipeline for a single CLI COMMAND against a device (§5)."""
+    from parseforge.pipeline import run_command_pipeline
+
+    run_command_pipeline(
+        " ".join(command),
+        key_fields={"vendor": vendor, "family": family, "os": os_, "version": version},
+    )
+
+
+if __name__ == "__main__":
+    main()

parseforge-0.2.1/parseforge/generation.py

@@ -0,0 +1,40 @@
+"""Generation stage — LLM call and template extraction (SPEC.md §5 steps 5-6).
+
+Writes raw-llm-response.txt / usage.txt (step 5), then extracts and
+cleans the template into raw-template / template.textfsm (step 6).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class GenerationResult:
+    raw_response: str
+    usage: str
+    raw_template: str
+    template: str
+
+
+class Generator(Protocol):
+    """An LLM backend capable of producing a TextFSM template from sample input.
+
+    ``prior_context`` carries additional samples of the same command when
+    running in Mode 1 (batch) per §4.
+    """
+
+    def generate(
+        self, input_text: str, prior_context: list[str] | None = None
+    ) -> GenerationResult: ...
+
+
+def extract_template(raw_response: str) -> str:
+    """Pull the template body out of a raw LLM response (step 6, pre-cleanup)."""
+    raise NotImplementedError("extraction rule TBD — see SPEC.md §5 step 6")
+
+
+def clean_template(raw_template: str) -> str:
+    """Normalize an extracted template into a valid template.textfsm body."""
+    raise NotImplementedError("cleanup rule TBD — see SPEC.md §5 step 6")

parseforge-0.2.1/parseforge/naming/__init__.py

@@ -0,0 +1,15 @@
+from .assemble import pattern_to_cli_name
+from .cache import DEFAULT_INDEX_PATH, NameIndex
+from .llm import CliContext, RegexBuilder, UnimplementedRegexBuilder, build_prompt
+from .resolver import cli_name
+
+__all__ = [
+    "cli_name",
+    "CliContext",
+    "RegexBuilder",
+    "UnimplementedRegexBuilder",
+    "build_prompt",
+    "NameIndex",
+    "DEFAULT_INDEX_PATH",
+    "pattern_to_cli_name",
+]

parseforge-0.2.1/parseforge/naming/assemble.py

@@ -0,0 +1,35 @@
+"""Turn a validated regex pattern into a canonical, kebab-case cli-name.
+
+Fixed-text segments pass through as-is; named capture groups
+(``(?P<var1>...)``) become their group name. Tokens are joined with
+``-`` and lowercased, matching SPEC.md §2's naming convention.
+"""
+
+from __future__ import annotations
+
+import re
+
+_LEADING_FLAGS = re.compile(r"^\(\?[a-zA-Z]+\)")
+_NAMED_GROUP = re.compile(r"^\(\?P<(?P<name>\w+)>.*\)$")
+_ESCAPED_CHAR = re.compile(r"\\(.)")
+
+
+def pattern_to_cli_name(pattern: str) -> str:
+    """
+    >>> pattern_to_cli_name(r"(?i)show\\s+version")
+    'show-version'
+    >>> pattern_to_cli_name(r"(?i)show\\s+interface\\s+(?P<var1>\\S+)\\s+status")
+    'show-interface-var1-status'
+    """
+    body = _LEADING_FLAGS.sub("", pattern)
+    tokens = [t for t in re.split(r"\\s\+", body) if t.strip()]
+
+    parts: list[str] = []
+    for token in tokens:
+        token = token.strip()
+        group = _NAMED_GROUP.match(token)
+        if group:
+            parts.append(group.group("name"))
+        else:
+            parts.append(_ESCAPED_CHAR.sub(r"\1", token))
+    return "-".join(parts).lower()

parseforge-0.2.1/parseforge/naming/cache.py

@@ -0,0 +1,48 @@
+"""On-disk cli-name -> regex index, so a CLI is only ever sent to the LLM once.
+
+Persisted as a flat JSON dict (SPEC.md naming discussion):
+    {"show-version": "(?i)show\\s+version", ...}
+
+The cli-name segment is vendor/OS-agnostic by design — vendor, family, os,
+and version are already distinguishing directory levels in the storage
+layout (SPEC.md §3), so a single global index is intentional rather than
+one index per device context.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+DEFAULT_INDEX_PATH = Path.home() / ".parseforge" / ".cli-name.json"
+
+
+class NameIndex:
+    """Loads, queries, and persists the cli-name -> regex mapping."""
+
+    def __init__(self, path: Path = DEFAULT_INDEX_PATH) -> None:
+        self.path = path
+        self._entries: dict[str, str] = self._load()
+
+    def _load(self) -> dict[str, str]:
+        if not self.path.exists():
+            return {}
+        return json.loads(self.path.read_text(encoding="utf-8"))
+
+    def match(self, command: str) -> str | None:
+        """Return the cli-name of the first stored pattern that fully matches
+        ``command``, or None if this command hasn't been named before."""
+        for cli_name, pattern in self._entries.items():
+            if re.fullmatch(pattern, command):
+                return cli_name
+        return None
+
+    def add(self, cli_name: str, pattern: str) -> None:
+        self._entries[cli_name] = pattern
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(
+            json.dumps(self._entries, indent=2, sort_keys=True), encoding="utf-8"
+        )

parseforge-0.2.1/parseforge/naming/llm.py

@@ -0,0 +1,62 @@
+"""LLM-driven regex construction for a CLI command.
+
+Only called on a cache miss (see resolver.py) — once a command's pattern
+is in the index, matching is pure regex and costs no tokens.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+PROMPT_TEMPLATE = """\
+This is CLI "{command}" of {vendor} {family} {os} version {version}.
+Create a regex pattern to match it:
+- If a token is fixed text, use it as-is.
+- If a token is variable text, use a regex to match it, and name the \
+capture group var1, var2, ... in left-to-right order of appearance.
+- Use \\s+ to match whitespace between tokens.
+
+Example 1: "show version" -> "show" and "version" are fixed text ->
+r"(?i)show\\s+version"
+
+Example 2: "show interface Ge1.1 status" -> "show", "interface", and \
+"status" are fixed text ->
+r"(?i)show\\s+interface\\s+(?P<var1>\\S+)\\s+status"
+
+Respond with only the regex pattern, nothing else.
+"""
+
+
+@dataclass(frozen=True)
+class CliContext:
+    vendor: str
+    family: str
+    os: str
+    version: str
+
+
+def build_prompt(command: str, context: CliContext) -> str:
+    return PROMPT_TEMPLATE.format(
+        command=command,
+        vendor=context.vendor,
+        family=context.family,
+        os=context.os,
+        version=context.version,
+    )
+
+
+class RegexBuilder(Protocol):
+    """An LLM backend that turns a raw CLI command into a regex pattern."""
+
+    def build_pattern(self, command: str, context: CliContext) -> str: ...
+
+
+class UnimplementedRegexBuilder:
+    """Default RegexBuilder — no LLM client is wired into the scaffold yet."""
+
+    def build_pattern(self, command: str, context: CliContext) -> str:
+        raise NotImplementedError(
+            "no RegexBuilder configured — wire an LLM client that sends "
+            "build_prompt(command, context) and returns the regex pattern"
+        )

parseforge-0.2.1/parseforge/naming/resolver.py

@@ -0,0 +1,45 @@
+"""Resolve a raw CLI command to its canonical cli-name, LLM-backed with caching.
+
+Flow:
+1. Check the on-disk index (:mod:`parseforge.naming.cache`) for a stored
+   pattern that already matches this command — no LLM call on a hit.
+2. On a miss, ask the LLM to build a regex for it
+   (:mod:`parseforge.naming.llm`).
+3. Validate the LLM's pattern actually matches its own source command via
+   ``re.fullmatch`` before trusting it.
+4. Assemble the cli-name from the pattern (:mod:`parseforge.naming.assemble`)
+   and persist the new cli-name -> pattern entry to the index.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from .assemble import pattern_to_cli_name
+from .cache import DEFAULT_INDEX_PATH, NameIndex
+from .llm import CliContext, RegexBuilder, UnimplementedRegexBuilder
+
+
+def cli_name(
+    command: str,
+    context: CliContext,
+    builder: RegexBuilder = UnimplementedRegexBuilder(),
+    index_path: Path = DEFAULT_INDEX_PATH,
+) -> str:
+    index = NameIndex(index_path)
+
+    cached = index.match(command)
+    if cached is not None:
+        return cached
+
+    pattern = builder.build_pattern(command, context)
+    if not re.fullmatch(pattern, command):
+        raise ValueError(
+            f"LLM-built pattern {pattern!r} does not match its own command {command!r}"
+        )
+
+    name = pattern_to_cli_name(pattern)
+    index.add(name, pattern)
+    index.save()
+    return name

parseforge-0.2.1/parseforge/paths.py

@@ -0,0 +1,68 @@
+"""Storage layout — three-tier promotion path resolution (SPEC.md §3).
+
+Shared path prefix under all three tiers:
+    <vendor>/<device-family>/<os>/<version>/<cli-name>/
+
+Use hyphenated OS family names (``ios-xe``, ``nx-os``) once multiple
+Cisco OS families share the tree, per §3's note.
+"""
+
+from __future__ import annotations
+
+import random
+import string
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+TRIALS = "trials"
+INTEGRATION = "integration"
+AUTHORITATIVE = "authoritative"
+
+
+@dataclass(frozen=True)
+class DeviceKey:
+    """Identifies a template family: vendor/device-family/os/version/cli-name."""
+
+    vendor: str
+    family: str
+    os: str
+    version: str
+    cli_name: str
+
+    def relative_path(self) -> Path:
+        return Path(self.vendor, self.family, self.os, self.version, self.cli_name)
+
+
+def tier_root(store_root: Path, tier: str) -> Path:
+    if tier not in (TRIALS, INTEGRATION, AUTHORITATIVE):
+        raise ValueError(f"unknown tier: {tier!r}")
+    return store_root / tier
+
+
+def tier_path(store_root: Path, tier: str, key: DeviceKey) -> Path:
+    """Resolve the directory for ``key`` under the given tier."""
+    return tier_root(store_root, tier) / key.relative_path()
+
+
+def new_run_id() -> str:
+    """Timestamp+shortid run identifier (§3.1) — chronological, collision-safe."""
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    shortid = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
+    return f"{timestamp}-{shortid}"
+
+
+def trial_run_dir(store_root: Path, key: DeviceKey, run_id: str | None = None) -> Path:
+    """Directory for a single trial run: trials/.../<cli-name>/<run-id>/."""
+    run_id = run_id or new_run_id()
+    return tier_path(store_root, TRIALS, key) / run_id
+
+
+def integration_result_dir(store_root: Path, key: DeviceKey) -> Path:
+    """The single common-result candidate directory for a cli-name (§3.2)."""
+    return tier_path(store_root, INTEGRATION, key) / "common-result"
+
+
+def authoritative_dir(store_root: Path, key: DeviceKey) -> Path:
+    """The approved, in-production directory for a cli-name (§3.3)."""
+    return tier_path(store_root, AUTHORITATIVE, key)

parseforge-0.2.1/parseforge/pipeline.py

@@ -0,0 +1,49 @@
+"""Pipeline orchestration (SPEC.md §4, §5).
+
+Mode 2 (loop: sample -> generate, per command) is the MVP per §4's
+recommendation. Mode 1 (batch: gather-then-generate) is a config flag
+that only changes the sampling stage — generation, storage, and
+validation stay identical between modes.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+from parseforge import naming, paths
+
+
+class Mode(str, Enum):
+    LOOP = "loop"  # MVP: sample -> generate, per command (§4)
+    BATCH = "batch"  # collect N samples per command before generation (§4)
+
+
+def run_command_pipeline(
+    command: str,
+    key_fields: dict,
+    regex_builder: naming.RegexBuilder = naming.UnimplementedRegexBuilder(),
+    mode: Mode = Mode.LOOP,
+) -> None:
+    """Execute steps 1-7 of the pipeline for a single CLI command.
+
+    1. Input intake is the caller's responsibility (device info, auth,
+       command list, mode selection already resolved into ``key_fields``,
+       which must supply vendor/family/os/version).
+    2. Name generation — :func:`parseforge.naming.cli_name` (LLM-backed,
+       cached — see :mod:`parseforge.naming`).
+    3. Path resolution — :mod:`parseforge.paths`.
+    4. Sampling — :mod:`parseforge.sampling`.
+    5-6. Generation — :mod:`parseforge.generation`.
+    7. Self-validation — :mod:`parseforge.validation`.
+
+    Steps 8-11 (integration selection, authoritative promotion, drift
+    monitoring, repeat/loop) run out-of-band across all trials for a
+    cli-name, not per single pipeline invocation — see
+    :mod:`parseforge.validation` and :mod:`parseforge.promotion`.
+    """
+    context = naming.CliContext(**key_fields)
+    name = naming.cli_name(command, context, builder=regex_builder)
+    key = paths.DeviceKey(cli_name=name, **key_fields)
+    raise NotImplementedError(
+        f"wire sampling -> generation -> validation for {key!r} (mode={mode.value})"
+    )

parseforge-0.2.1/parseforge/promotion.py

@@ -0,0 +1,43 @@
+"""Authoritative promotion and drift monitoring (SPEC.md §3.3, §5 steps 9-10)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class PromotionDecision(str, Enum):
+    AUTO_PROMOTED = "auto_promoted"
+    QUEUED_FOR_REVIEW = "queued_for_review"
+
+
+@dataclass(frozen=True)
+class PromotionGate:
+    """Confidence threshold gating auto-promotion vs. human review (§3.3, §7).
+
+    Left configurable per-project rather than hardcoded — see the open
+    question in SPEC.md §7.
+    """
+
+    match_rate_threshold: float = 1.0
+    min_sample_count: int = 1
+
+
+def decide_promotion(
+    match_rate: float, sample_count: int, gate: PromotionGate
+) -> PromotionDecision:
+    if (
+        sample_count >= gate.min_sample_count
+        and match_rate >= gate.match_rate_threshold
+    ):
+        return PromotionDecision.AUTO_PROMOTED
+    return PromotionDecision.QUEUED_FOR_REVIEW
+
+
+@dataclass(frozen=True)
+class DriftStatus:
+    match_rate: float
+    status: str  # "ok" | "drifting" | "superseded" — see SPEC.md §6
+
+    def breached(self, threshold: float) -> bool:
+        return self.match_rate < threshold

parseforge-0.2.1/parseforge/sampling.py

@@ -0,0 +1,36 @@
+"""Sampling stage — connect to a device and capture raw CLI output (SPEC.md §5 step 4).
+
+Kept as a separable stage so that Mode 1 (batch: collect N samples per
+command before generation) and Mode 2 (loop: sample-then-generate per
+command) share the same generation/storage/validation stages and differ
+only in how many samples this stage collects before handing off — see
+§4.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class DeviceConnection:
+    host: str
+    username: str
+    password: str
+    device_type: str  # e.g. "cisco_ios", "cisco_nxos"
+
+
+class Sampler(Protocol):
+    """A backend capable of running a command on a device and returning raw text.
+
+    A Netmiko-backed implementation is the expected default; see the
+    optional ``sampling`` extra in pyproject.toml.
+    """
+
+    def run_command(self, connection: DeviceConnection, command: str) -> str: ...
+
+
+def sample(sampler: Sampler, connection: DeviceConnection, command: str) -> str:
+    """Run ``command`` against ``connection`` and return raw output for input.txt."""
+    return sampler.run_command(connection, command)

parseforge-0.2.1/parseforge/validation.py

@@ -0,0 +1,50 @@
+"""Validation — self-validation (SPEC.md §5 step 7) and cross-validation
+for integration selection (§3.2, §5 step 8).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ParseResult:
+    records: list[dict]
+    errors: list[str]
+
+    @property
+    def passed(self) -> bool:
+        return not self.errors
+
+
+def parse(template_text: str, input_text: str) -> ParseResult:
+    """Run a template.textfsm against an input.txt sample.
+
+    Used both for self-validation (a template against its own sample,
+    step 7) and cross-validation (every candidate against every known
+    sample for a cli-name, step 8).
+    """
+    raise NotImplementedError("wrap textfsm.TextFSM — see SPEC.md §5 steps 7-8")
+
+
+@dataclass(frozen=True)
+class CandidateScore:
+    run_id: str
+    match_rate: float
+    failed_samples: list[str]
+
+
+def select_common_result(
+    candidates: dict[str, str], samples: dict[str, str]
+) -> CandidateScore:
+    """Cross-validate every candidate template against every known sample.
+
+    ``candidates`` maps trial run_id -> template.textfsm text.
+    ``samples`` maps sample id -> input.txt text.
+
+    Promotes whichever template parses the largest share of *all* known
+    samples correctly (not just the sample it was generated from) — an
+    overfit template that only matches its own source sample must lose
+    to a template that generalizes, per §3.2.
+    """
+    raise NotImplementedError("scoring/selection rule — see SPEC.md §3.2")

parseforge-0.2.1/parseforge.egg-info/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: parseforge
+Version: 0.2.1
+Summary: LLM-driven pipeline that forges, validates, and promotes TextFSM templates from network CLI output
+Author-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
+Maintainer-email: Tuyen Mathew Duong <tuyen@geekstrident.com>
+License: MIT
+Keywords: textfsm,network automation,cli parsing,template generator,ai,llm,netmiko,device output parsing
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: textfsm>=1.1.0
+Provides-Extra: sampling
+Requires-Dist: netmiko>=4.0; extra == "sampling"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: mypy>=1.10.0; extra == "dev"
+Dynamic: license-file
+
+# ParseForge
+
+LLM-driven pipeline that forges, cross-validates, and promotes [TextFSM](https://github.com/google/textfsm)
+templates from network device CLI output.
+
+Full design plan: [SPEC.md](SPEC.md).
+
+## Layout
+
+```
+parseforge/
+  naming.py       CLI command -> canonical cli-name (SPEC §2)
+  paths.py        trials/integration/authoritative path resolution (SPEC §3)
+  sampling.py     device connection + command capture (SPEC §5 step 4)
+  generation.py   LLM call + template extraction (SPEC §5 steps 5-6)
+  validation.py   self-validation + cross-validation scoring (SPEC §5 steps 7-8)
+  promotion.py    authoritative promotion gate + drift status (SPEC §3.3, §5 steps 9-10)
+  pipeline.py     orchestrates the above; Mode LOOP (MVP) vs Mode BATCH (SPEC §4)
+  cli/            `parseforge` command-line entry point
+
+store/            runtime template tree: trials/ -> integration/ -> authoritative/
+                  (generated at runtime, not checked in — see .gitignore)
+```
+
+`naming.py` and `paths.py` are implemented and tested. `sampling.py`, `generation.py`,
+`validation.py`, `promotion.py`, and `pipeline.py` are stub interfaces — the SPEC.md
+section cited in each docstring is the next thing to implement.
+
+## Development
+
+```
+pip install -e ".[dev,sampling]"
+pytest
+```
+
+## CLI
+
+```
+parseforge name show interface GE1.1 status
+# -> show-interface-var1-status
+```

parseforge-0.2.1/parseforge.egg-info/SOURCES.txt

@@ -0,0 +1,25 @@
+LICENSE
+MANIFEST.in
+README.md
+SPEC.md
+pyproject.toml
+parseforge/__init__.py
+parseforge/generation.py
+parseforge/paths.py
+parseforge/pipeline.py
+parseforge/promotion.py
+parseforge/sampling.py
+parseforge/validation.py
+parseforge.egg-info/PKG-INFO
+parseforge.egg-info/SOURCES.txt
+parseforge.egg-info/dependency_links.txt
+parseforge.egg-info/entry_points.txt
+parseforge.egg-info/requires.txt
+parseforge.egg-info/top_level.txt
+parseforge/cli/__init__.py
+parseforge/cli/main.py
+parseforge/naming/__init__.py
+parseforge/naming/assemble.py
+parseforge/naming/cache.py
+parseforge/naming/llm.py
+parseforge/naming/resolver.py

parseforge-0.2.1/parseforge.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

parseforge-0.2.1/parseforge.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+parseforge = parseforge.cli.main:main

parseforge-0.2.1/parseforge.egg-info/requires.txt

@@ -0,0 +1,13 @@
+click>=8.1
+PyYAML>=6.0
+textfsm>=1.1.0
+
+[dev]
+pytest>=8.0.0
+pytest-cov>=5.0.0
+ruff>=0.4.0
+black>=24.0.0
+mypy>=1.10.0
+
+[sampling]
+netmiko>=4.0

parseforge-0.2.1/parseforge.egg-info/top_level.txt

@@ -0,0 +1 @@
+parseforge

parseforge-0.2.1/pyproject.toml

@@ -0,0 +1,98 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "parseforge"
+version = "0.2.1"
+description = "LLM-driven pipeline that forges, validates, and promotes TextFSM templates from network CLI output"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+
+authors = [
+    { name = "Tuyen Mathew Duong", email = "tuyen@geekstrident.com" }
+]
+
+maintainers = [
+    { name = "Tuyen Mathew Duong", email = "tuyen@geekstrident.com" }
+]
+
+dependencies = [
+    "click>=8.1",
+    "PyYAML>=6.0",
+    "textfsm>=1.1.0",
+]
+
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: System Administrators",
+
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Text Processing",
+    "Topic :: Utilities",
+
+    "Operating System :: OS Independent",
+]
+
+keywords = [
+    "textfsm",
+    "network automation",
+    "cli parsing",
+    "template generator",
+    "ai",
+    "llm",
+    "netmiko",
+    "device output parsing",
+]
+
+[project.optional-dependencies]
+sampling = ["netmiko>=4.0"]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=5.0.0",
+    "ruff>=0.4.0",
+    "black>=24.0.0",
+    "mypy>=1.10.0",
+]
+
+[project.scripts]
+parseforge = "parseforge.cli.main:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["parseforge*"]
+exclude = ["tests*", "store*"]
+
+[tool.pytest.ini_options]
+addopts = "-ra --cov=parseforge --cov-report=term-missing"
+testpaths = ["tests"]
+pythonpath = ["."]
+
+[tool.coverage.report]
+# Raise as stub modules (sampling/generation/validation/promotion/pipeline)
+# get real implementations — see README.md.
+fail_under = 0
+
+[tool.black]
+line-length = 88
+target-version = ["py39"]
+
+[tool.ruff]
+line-length = 88
+fix = true
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I"]

parseforge-0.2.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+