ntermqt 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

nterm/parser/ntc_download_dialog.py

@@ -0,0 +1,372 @@
+
+import sys
+import json
+import sqlite3
+import hashlib
+import traceback
+from pathlib import Path
+from datetime import datetime
+from typing import Optional, List, Dict, Any
+
+from PyQt6.QtWidgets import (
+    QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
+    QTextEdit, QLineEdit, QPushButton, QLabel, QSplitter, QTableWidget,
+    QTableWidgetItem, QTabWidget, QGroupBox, QSpinBox, QCheckBox,
+    QFileDialog, QMessageBox, QComboBox, QDialog, QDialogButtonBox,
+    QFormLayout, QHeaderView, QAbstractItemView, QMenu, QInputDialog,
+    QStatusBar, QToolBar, QFrame
+)
+from PyQt6.QtCore import Qt, QThread, pyqtSignal, QSize
+from PyQt6.QtGui import QFont, QAction, QIcon, QColor, QPalette, QShortcut, QKeySequence
+
+import textfsm
+import io
+from collections import defaultdict
+
+
+# Try to import requests for NTC GitHub download
+REQUESTS_AVAILABLE = False
+try:
+    import requests
+
+    REQUESTS_AVAILABLE = True
+except ImportError:
+    pass
+
+# Try to import the engine, but don't fail if not available (manual mode still works)
+TFSM_ENGINE_AVAILABLE = False
+try:
+    from nterm.parser.tfsm_fire import TextFSMAutoEngine
+
+    TFSM_ENGINE_AVAILABLE = True
+except ImportError:
+    try:
+        from .tfsm_fire import TextFSMAutoEngine
+
+        TFSM_ENGINE_AVAILABLE = True
+    except ImportError:
+        pass
+
+GITHUB_API_URL = "https://api.github.com/repos/networktocode/ntc-templates/contents/ntc_templates/templates"
+GITHUB_RAW_BASE = "https://raw.githubusercontent.com/networktocode/ntc-templates/master/ntc_templates/templates"
+
+VENDOR_PREFIXES = [
+    'cisco', 'arista', 'juniper', 'hp', 'dell', 'paloalto', 'fortinet',
+    'brocade', 'extreme', 'huawei', 'mikrotik', 'ubiquiti', 'vmware',
+    'checkpoint', 'alcatel', 'avaya', 'ruckus', 'f5', 'a10', 'linux',
+    'yamaha', 'zyxel', 'enterasys', 'adtran', 'ciena', 'nokia', 'watchguard'
+]
+
+def extract_platform(filename: str) -> str:
+    """Extract platform name from template filename."""
+    name = filename.replace('.textfsm', '')
+    parts = name.split('_')
+    if len(parts) >= 2 and parts[0] in VENDOR_PREFIXES:
+        return f"{parts[0]}_{parts[1]}"
+    if parts[0] in VENDOR_PREFIXES:
+        return parts[0]
+    return parts[0]
+
+
+def get_package_db_path() -> Path:
+    """Database is in same directory as this module."""
+    return Path(__file__).parent / "tfsm_templates.db"
+
+
+class NTCDownloadWorker(QThread):
+    """Worker thread for downloading NTC templates"""
+    progress = pyqtSignal(int, int, str)  # current, total, status
+    finished = pyqtSignal(dict)  # stats dict
+    error = pyqtSignal(str)
+
+    def __init__(self, platforms: list, db_path: str, replace: bool = False):
+        super().__init__()
+        self.platforms = platforms
+        self.db_path = db_path or str(get_package_db_path())
+        self.replace = replace
+        self.templates_to_download = []
+
+    def run(self):
+        try:
+            # Fetch template list
+            response = requests.get(GITHUB_API_URL, timeout=30)
+            response.raise_for_status()
+
+            files = response.json()
+            all_templates = [f for f in files if f['name'].endswith('.textfsm')]
+
+            # Group by platform
+            platforms_map = defaultdict(list)
+            for t in all_templates:
+                platform = extract_platform(t['name'])
+                platforms_map[platform].append(t)
+
+            # Filter to selected platforms
+            for platform in self.platforms:
+                if platform in platforms_map:
+                    self.templates_to_download.extend(platforms_map[platform])
+
+            if not self.templates_to_download:
+                self.finished.emit({'imported': 0, 'updated': 0, 'skipped': 0, 'errors': 0})
+                return
+
+            # Connect to database
+            conn = sqlite3.connect(self.db_path)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS templates (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    cli_command TEXT UNIQUE,
+                    cli_content TEXT,
+                    textfsm_content TEXT,
+                    textfsm_hash TEXT,
+                    source TEXT,
+                    created TEXT
+                )
+            """)
+            cursor = conn.cursor()
+
+            stats = {'imported': 0, 'updated': 0, 'skipped': 0, 'errors': 0}
+            total = len(self.templates_to_download)
+
+            for i, template in enumerate(self.templates_to_download, 1):
+                name = template['name']
+                cli_command = name.replace('.textfsm', '')
+
+                try:
+                    # Download content
+                    url = f"{GITHUB_RAW_BASE}/{name}"
+                    resp = requests.get(url, timeout=30)
+                    resp.raise_for_status()
+                    content = resp.text
+
+                    textfsm_hash = hashlib.md5(content.encode()).hexdigest()
+                    created = datetime.now().isoformat()
+
+                    # Check if exists
+                    cursor.execute("SELECT textfsm_hash FROM templates WHERE cli_command = ?", (cli_command,))
+                    existing = cursor.fetchone()
+
+                    if existing:
+                        if self.replace and existing[0] != textfsm_hash:
+                            cursor.execute("""
+                                UPDATE templates 
+                                SET textfsm_content = ?, textfsm_hash = ?, source = ?, created = ?
+                                WHERE cli_command = ?
+                            """, (content, textfsm_hash, "ntc-templates", created, cli_command))
+                            stats['updated'] += 1
+                            status = "U"
+                        else:
+                            stats['skipped'] += 1
+                            status = "."
+                    else:
+                        cursor.execute("""
+                            INSERT INTO templates (cli_command, cli_content, textfsm_content, textfsm_hash, source, created)
+                            VALUES (?, ?, ?, ?, ?, ?)
+                        """, (cli_command, "", content, textfsm_hash, "ntc-templates", created))
+                        stats['imported'] += 1
+                        status = "+"
+
+                    self.progress.emit(i, total, f"{status} {cli_command}")
+
+                except Exception as e:
+                    stats['errors'] += 1
+                    self.progress.emit(i, total, f"E {cli_command}: {str(e)[:30]}")
+
+            conn.commit()
+            conn.close()
+            self.finished.emit(stats)
+
+        except Exception as e:
+            traceback.print_exc()
+            self.error.emit(str(e))
+
+
+
+class NTCDownloadDialog(QDialog):
+    """Dialog for selecting and downloading NTC templates from GitHub"""
+
+    def __init__(self, parent=None, db_path: str = "tfsm_templates.db"):
+        super().__init__(parent)
+        self.db_path = db_path
+        self.platforms = {}
+        self.setWindowTitle("Download NTC Templates from GitHub")
+        self.setMinimumSize(600, 500)
+        self.setup_ui()
+
+    def setup_ui(self):
+        layout = QVBoxLayout(self)
+
+        # Info label
+        info = QLabel("Download TextFSM templates directly from networktocode/ntc-templates GitHub repository.")
+        info.setWordWrap(True)
+        layout.addWidget(info)
+
+        # Fetch button
+        fetch_layout = QHBoxLayout()
+        self.fetch_btn = QPushButton("Fetch Available Platforms")
+        self.fetch_btn.clicked.connect(self.fetch_platforms)
+        fetch_layout.addWidget(self.fetch_btn)
+        fetch_layout.addStretch()
+        layout.addLayout(fetch_layout)
+
+        # Platform list
+        list_group = QGroupBox("Available Platforms")
+        list_layout = QVBoxLayout(list_group)
+
+        # Select all / none buttons
+        select_layout = QHBoxLayout()
+        select_all_btn = QPushButton("Select All")
+        select_all_btn.setProperty("secondary", True)
+        select_all_btn.clicked.connect(self.select_all)
+        select_layout.addWidget(select_all_btn)
+
+        select_none_btn = QPushButton("Select None")
+        select_none_btn.setProperty("secondary", True)
+        select_none_btn.clicked.connect(self.select_none)
+        select_layout.addWidget(select_none_btn)
+
+        select_layout.addStretch()
+
+        self.status_label = QLabel("")
+        select_layout.addWidget(self.status_label)
+        list_layout.addLayout(select_layout)
+
+        # Platform table with checkboxes
+        self.platform_table = QTableWidget()
+        self.platform_table.setColumnCount(3)
+        self.platform_table.setHorizontalHeaderLabels(["Select", "Platform", "Templates"])
+        self.platform_table.horizontalHeader().setStretchLastSection(True)
+        self.platform_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
+        self.platform_table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
+        list_layout.addWidget(self.platform_table)
+
+        layout.addWidget(list_group)
+
+        # Options
+        options_layout = QHBoxLayout()
+        self.replace_check = QCheckBox("Replace existing templates if content changed")
+        options_layout.addWidget(self.replace_check)
+        options_layout.addStretch()
+        layout.addLayout(options_layout)
+
+        # Progress
+        self.progress_label = QLabel("")
+        layout.addWidget(self.progress_label)
+
+        # Buttons
+        btn_layout = QHBoxLayout()
+        btn_layout.addStretch()
+
+        self.download_btn = QPushButton("Download Selected")
+        self.download_btn.clicked.connect(self.start_download)
+        self.download_btn.setEnabled(False)
+        btn_layout.addWidget(self.download_btn)
+
+        close_btn = QPushButton("Close")
+        close_btn.setProperty("secondary", True)
+        close_btn.clicked.connect(self.accept)
+        btn_layout.addWidget(close_btn)
+
+        layout.addLayout(btn_layout)
+
+    def fetch_platforms(self):
+        """Fetch available platforms from GitHub"""
+        if not REQUESTS_AVAILABLE:
+            QMessageBox.critical(self, "Error", "requests library not available.\nInstall with: pip install requests")
+            return
+
+        self.fetch_btn.setEnabled(False)
+        self.status_label.setText("Fetching from GitHub...")
+        QApplication.processEvents()
+
+        try:
+            response = requests.get(GITHUB_API_URL, timeout=30)
+            response.raise_for_status()
+
+            files = response.json()
+            templates = [f for f in files if f['name'].endswith('.textfsm')]
+
+            # Group by platform
+            self.platforms = defaultdict(list)
+            for t in templates:
+                platform = extract_platform(t['name'])
+                self.platforms[platform].append(t)
+
+            # Populate table
+            self.platform_table.setRowCount(len(self.platforms))
+            for row, (platform, tmpl_list) in enumerate(sorted(self.platforms.items(), key=lambda x: -len(x[1]))):
+                # Checkbox
+                checkbox = QCheckBox()
+                self.platform_table.setCellWidget(row, 0, checkbox)
+
+                # Platform name
+                self.platform_table.setItem(row, 1, QTableWidgetItem(platform))
+
+                # Template count
+                self.platform_table.setItem(row, 2, QTableWidgetItem(str(len(tmpl_list))))
+
+            self.status_label.setText(f"Found {len(templates)} templates across {len(self.platforms)} platforms")
+            self.download_btn.setEnabled(True)
+
+        except Exception as e:
+            traceback.print_exc()
+            QMessageBox.critical(self, "Error", f"Failed to fetch platforms:\n{str(e)}")
+            self.status_label.setText("Fetch failed")
+
+        self.fetch_btn.setEnabled(True)
+
+    def select_all(self):
+        for row in range(self.platform_table.rowCount()):
+            checkbox = self.platform_table.cellWidget(row, 0)
+            if checkbox:
+                checkbox.setChecked(True)
+
+    def select_none(self):
+        for row in range(self.platform_table.rowCount()):
+            checkbox = self.platform_table.cellWidget(row, 0)
+            if checkbox:
+                checkbox.setChecked(False)
+
+    def get_selected_platforms(self) -> list:
+        selected = []
+        for row in range(self.platform_table.rowCount()):
+            checkbox = self.platform_table.cellWidget(row, 0)
+            if checkbox and checkbox.isChecked():
+                item = self.platform_table.item(row, 1)
+                if item:
+                    selected.append(item.text())
+        return selected
+
+    def start_download(self):
+        selected = self.get_selected_platforms()
+        if not selected:
+            QMessageBox.warning(self, "Warning", "Please select at least one platform")
+            return
+
+        self.download_btn.setEnabled(False)
+        self.fetch_btn.setEnabled(False)
+        self.progress_label.setText("Starting download...")
+
+        self.worker = NTCDownloadWorker(selected, self.db_path, self.replace_check.isChecked())
+        self.worker.progress.connect(self.update_progress)
+        self.worker.finished.connect(self.download_finished)
+        self.worker.error.connect(self.download_error)
+        self.worker.start()
+
+    def update_progress(self, current: int, total: int, status: str):
+        self.progress_label.setText(f"[{current}/{total}] {status}")
+
+    def download_finished(self, stats: dict):
+        self.download_btn.setEnabled(True)
+        self.fetch_btn.setEnabled(True)
+
+        msg = f"Download Complete!\n\nImported: {stats['imported']}\nUpdated: {stats['updated']}\nSkipped: {stats['skipped']}\nErrors: {stats['errors']}"
+        self.progress_label.setText(
+            f"Done: {stats['imported']} imported, {stats['updated']} updated, {stats['skipped']} skipped")
+        QMessageBox.information(self, "Download Complete", msg)
+
+    def download_error(self, error: str):
+        self.download_btn.setEnabled(True)
+        self.fetch_btn.setEnabled(True)
+        self.progress_label.setText("Download failed")
+        QMessageBox.critical(self, "Error", f"Download failed:\n{error}")

nterm/parser/tfsm_engine.py

@@ -0,0 +1,246 @@
+"""
+TextFSM Validation Engine - Wrapper around tfsm_fire.TextFSMAutoEngine.
+
+Validates collected output against TextFSM templates.
+Only output with score > 0 is considered valid.
+
+Usage:
+    engine = ValidationEngine(db_path="path/to/tfsm_templates.db")
+    result = engine.validate(output, filter_string="cisco_ios_show_version")
+
+    if result.is_valid:
+        print(f"Template: {result.template}")
+        print(f"Score: {result.score}")
+        print(f"Records: {len(result.parsed_data)}")
+    else:
+        print("Invalid output - no matching template")
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+# Import the actual engine from core
+from tfsm_fire import TextFSMAutoEngine
+
+
+@dataclass
+class ValidationResult:
+    """Result of output validation."""
+    is_valid: bool
+    template: Optional[str] = None
+    parsed_data: Optional[List[Dict]] = None
+    score: float = 0.0
+    error: Optional[str] = None
+
+    @property
+    def record_count(self) -> int:
+        """Number of records parsed."""
+        return len(self.parsed_data) if self.parsed_data else 0
+
+
+class ValidationEngine:
+    """
+    TextFSM-based output validation engine.
+
+    Wraps tfsm_fire.TextFSMAutoEngine for use in the collection pipeline.
+    Output must achieve a score > 0 to be considered valid (no cruft).
+
+    Attributes:
+        db_path: Path to TextFSM templates database
+        min_score: Minimum score for valid output (default: 0.01)
+        verbose: Enable verbose logging
+    """
+
+    def __init__(
+            self,
+            db_path: Optional[str] = None,
+            min_score: float = 0.01,
+            verbose: bool = False,
+    ):
+        """
+        Initialize validation engine.
+
+        Args:
+            db_path: Path to tfsm_templates.db. If None, uses default location.
+            min_score: Minimum score to consider output valid.
+            verbose: Enable verbose output.
+        """
+        # Default database location - check multiple places
+        if db_path is None:
+            possible_paths = [
+                Path(__file__).parent.parent / "core" / "tfsm_templates.db",
+                Path.home() / ".vcollector" / "tfsm_templates.db",
+            ]
+            for p in possible_paths:
+                if p.exists():
+                    db_path = str(p)
+                    break
+            else:
+                raise FileNotFoundError(
+                    f"TextFSM template database not found. Searched:\n"
+                    f"  - {possible_paths[0]}\n"
+                    f"  - {possible_paths[1]}\n"
+                )
+
+        self.db_path = db_path
+        self.min_score = min_score
+        self.verbose = verbose
+
+        # Verify database exists
+        if not Path(db_path).exists():
+            raise FileNotFoundError(f"TextFSM template database not found: {db_path}")
+
+        # Initialize the actual engine
+        self._engine = TextFSMAutoEngine(db_path, verbose=verbose)
+
+    def _clean_output(self, raw_output: str) -> str:
+        """
+        Clean raw CLI output for TextFSM parsing.
+
+        Removes:
+        - Preamble lines (terminal length, pagination messages)
+        - Command echo (hostname#show command)
+        - Trailing prompts
+
+        Args:
+            raw_output: Raw output from SSH session
+
+        Returns:
+            Cleaned output suitable for TextFSM parsing
+        """
+        import re
+
+        lines = raw_output.split('\n')
+        cleaned_lines = []
+        found_output_start = False
+
+        # Common preamble patterns to skip
+        preamble_patterns = [
+            r'^terminal\s+(length|width)',
+            r'^pagination\s+disabled',
+            r'^screen-length\s+disable',
+            r'^\s*$',  # Empty lines at start
+        ]
+
+        # Command echo pattern: hostname#command or hostname>command
+        # Also matches: hostname(config)#, hostname(config-if)#, etc.
+        command_echo_pattern = r'^[\w\-\.]+[\#\>\$\)].*?(show|display|get)\s+'
+
+        # Trailing prompt pattern
+        trailing_prompt_pattern = r'^[\w\-\.]+[\#\>\$\)]\s*$'
+
+        for i, line in enumerate(lines):
+            line_stripped = line.strip()
+
+            # Skip empty lines at the start
+            if not found_output_start and not line_stripped:
+                continue
+
+            # Skip preamble lines
+            if not found_output_start:
+                is_preamble = False
+                for pattern in preamble_patterns:
+                    if re.match(pattern, line_stripped, re.IGNORECASE):
+                        is_preamble = True
+                        break
+
+                if is_preamble:
+                    continue
+
+                # Check for command echo line
+                if re.match(command_echo_pattern, line_stripped, re.IGNORECASE):
+                    found_output_start = True
+                    continue  # Skip the command echo itself
+
+                # If we get here without matching preamble or command echo,
+                # this is probably the start of actual output
+                found_output_start = True
+
+            # Skip trailing prompts
+            if re.match(trailing_prompt_pattern, line_stripped):
+                continue
+
+            cleaned_lines.append(line)
+
+        # Remove trailing empty lines
+        while cleaned_lines and not cleaned_lines[-1].strip():
+            cleaned_lines.pop()
+
+        return '\n'.join(cleaned_lines)
+
+    def validate(
+            self,
+            device_output: str,
+            filter_string: Optional[str] = None,
+    ) -> ValidationResult:
+        """
+        Validate device output against TextFSM templates.
+
+        Args:
+            device_output: Raw CLI output from device.
+            filter_string: Template filter (e.g., "cisco_ios_show_version").
+
+        Returns:
+            ValidationResult with validation status and parsed data.
+        """
+        if not device_output or not device_output.strip():
+            return ValidationResult(
+                is_valid=False,
+                error="Empty output"
+            )
+
+        try:
+            # Clean the output before validation
+            cleaned_output = self._clean_output(device_output)
+
+            if self.verbose:
+                print(f"[VALIDATION] Cleaned output ({len(cleaned_output)} chars):")
+                print(cleaned_output[:500] + "..." if len(cleaned_output) > 500 else cleaned_output)
+
+            # Use tfsm_fire engine to find best template
+            template, parsed_data, score = self._engine.find_best_template(
+                cleaned_output, filter_string
+            )
+
+            is_valid = score >= self.min_score and parsed_data is not None
+
+            return ValidationResult(
+                is_valid=is_valid,
+                template=template,
+                parsed_data=parsed_data,
+                score=score,
+            )
+
+        except Exception as e:
+            return ValidationResult(
+                is_valid=False,
+                error=str(e)
+            )
+
+    def list_templates(self, filter_string: Optional[str] = None) -> List[str]:
+        """List available templates matching filter."""
+        with self._engine.connection_manager.get_connection() as conn:
+            templates = self._engine.get_filtered_templates(conn, filter_string)
+            return [t['cli_command'] for t in templates]
+
+
+# Convenience function for simple validation
+def validate_output(
+        output: str,
+        filter_string: str,
+        db_path: Optional[str] = None,
+) -> ValidationResult:
+    """
+    Validate device output against TextFSM templates.
+
+    Args:
+        output: Raw CLI output from device.
+        filter_string: Template filter (e.g., "cisco_ios_show_version").
+        db_path: Path to template database (optional).
+
+    Returns:
+        ValidationResult with validation status.
+    """
+    engine = ValidationEngine(db_path=db_path)
+    return engine.validate(output, filter_string)