sfacts 2.3.0__tar.gz

sfacts-2.3.0/.env.template

@@ -0,0 +1,11 @@
+# NetBox Integration Configuration
+# Copy this file to .env and fill in your actual values
+
+# NetBox instance URL (include http:// or https://)
+NETBOX_URL=https://your-netbox-instance.com
+
+# NetBox API token (generate from NetBox Admin > API Tokens)
+NETBOX_TOKEN=your-api-token-here
+
+# Optional: Disable SSL verification for testing (not recommended for production)
+# NETBOX_VERIFY_SSL=false

sfacts-2.3.0/.gitignore

@@ -0,0 +1,144 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Ruff cache
+.ruff_cache/
+
+# Installer logs
+
+# IDE specific files
+.idea/
+.vscode/
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation & blueprints
+/site
+blueprints/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Project specific
+*.swp
+.DS_Store
+
+# uv
+.venv/
+
+# Output files
+output/
+
+# Claude Code worktrees and project data
+.claude/
+
+# Benchmark results
+.benchmarks/
+
+# GitLab CI/CD local runner cache
+.gitlab/
+
+# Netlab labs
+labs/

sfacts-2.3.0/.gitlab-ci-simple.yml

@@ -0,0 +1,85 @@
+# Simplified GitLab CI/CD Pipeline for SFacts
+# This version is tested and working
+
+image: python:3.11-slim
+
+stages:
+  - lint
+  - test
+  - build
+
+# Cache pip packages
+cache:
+  paths:
+    - .cache/pip
+  key: ${CI_COMMIT_REF_SLUG}
+
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+# Install dependencies before each job
+.install_deps: &install_deps
+  before_script:
+    - pip install --upgrade pip
+    - pip install -e .
+    - pip install ruff pytest pytest-cov pytest-mock mypy
+
+# ============================================================================
+# LINT STAGE
+# ============================================================================
+
+ruff-lint:
+  stage: lint
+  <<: *install_deps
+  script:
+    - echo "Running Ruff linter..."
+    - ruff check sfacts/ tests/ --statistics
+  allow_failure: false
+
+ruff-format:
+  stage: lint
+  <<: *install_deps
+  script:
+    - echo "Checking code formatting..."
+    - ruff format --check sfacts/ tests/
+  allow_failure: false
+
+# ============================================================================
+# TEST STAGE
+# ============================================================================
+
+unit-tests:
+  stage: test
+  <<: *install_deps
+  script:
+    - echo "Running unit tests..."
+    - pytest tests/unit/ -v --ignore=tests/integration --ignore=tests/e2e --ignore=tests/unit/test_netbox_sync.py --ignore=tests/unit/test_netbox_bulk_sync.py --ignore=tests/unit/test_tasks/test_netbox_task.py --cov=sfacts --cov-report=term --cov-report=xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+    paths:
+      - htmlcov/
+    expire_in: 30 days
+
+# ============================================================================
+# BUILD STAGE (only on main/tags)
+# ============================================================================
+
+build-package:
+  stage: build
+  before_script:
+    - pip install build
+  script:
+    - echo "Building Python package..."
+    - python -m build
+    - ls -lh dist/
+  artifacts:
+    paths:
+      - dist/
+    expire_in: 30 days
+  only:
+    - main
+    - tags

sfacts-2.3.0/.gitlab-ci.yml

@@ -0,0 +1,153 @@
+# GitLab CI/CD Pipeline for SFacts
+# Modern CI/CD with uv: fast, reproducible Python builds
+
+image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim
+
+stages:
+  - lint
+  - test
+  - security
+  - build
+  - deploy
+
+# Cache uv packages and virtual environment
+cache:
+  key:
+    files:
+      - uv.lock
+  paths:
+    - .uv-cache/
+    - .venv/
+
+variables:
+  UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache"
+  UV_LINK_MODE: "copy"  # Required for Docker builds
+
+# Template for uv-based jobs
+.uv_sync: &uv_sync
+  before_script:
+    - echo "Syncing dependencies with uv..."
+    - uv sync --frozen --all-groups
+
+# ============================================================================
+# LINT STAGE: Code quality checks
+# ============================================================================
+
+ruff-lint:
+  stage: lint
+  <<: *uv_sync
+  script:
+    - echo "Running Ruff linter..."
+    - uv run ruff check sfacts/ tests/ --statistics || true
+    - echo "Checking for critical errors only..."
+    - uv run ruff check sfacts/ tests/ --select=E9,F63,F7,F82 --statistics
+  allow_failure: false
+
+ruff-format-check:
+  stage: lint
+  <<: *uv_sync
+  script:
+    - echo "Checking code formatting with Ruff..."
+    - uv run ruff format --check sfacts/ tests/
+  allow_failure: false
+
+mypy-type-check:
+  stage: lint
+  <<: *uv_sync
+  script:
+    - echo "Running MyPy type checker..."
+    - uv run mypy sfacts/ --ignore-missing-imports
+  allow_failure: true  # Allow failure initially as type hints are being added
+
+# ============================================================================
+# TEST STAGE: Run test suites
+# ============================================================================
+
+unit-tests:
+  stage: test
+  <<: *uv_sync
+  script:
+    - echo "Running unit tests..."
+    - uv run pytest tests/unit/ -v --ignore=tests/integration --ignore=tests/e2e --ignore=tests/unit/test_netbox_sync.py --ignore=tests/unit/test_netbox_bulk_sync.py --ignore=tests/unit/test_tasks/test_netbox_task.py --cov=sfacts --cov-report=term --cov-report=xml --cov-report=html
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+    paths:
+      - htmlcov/
+      - coverage.xml
+    expire_in: 30 days
+  allow_failure: false
+
+# ============================================================================
+# SECURITY STAGE: Security scanning
+# ============================================================================
+
+dependency-scan:
+  stage: security
+  <<: *uv_sync
+  script:
+    - echo "Scanning dependencies for known vulnerabilities..."
+    - uv run safety check --json || true  # Don't fail pipeline on vulnerabilities yet
+  allow_failure: true
+  artifacts:
+    reports:
+      dependency_scanning: gl-dependency-scanning-report.json
+    expire_in: 30 days
+  only:
+    - main
+    - develop
+
+bandit-security-scan:
+  stage: security
+  <<: *uv_sync
+  script:
+    - echo "Running Bandit security linter..."
+    - uv run bandit -r sfacts/ -f json -o bandit-report.json --severity-level medium --confidence-level high || true
+    - uv run bandit -r sfacts/ -f screen --severity-level medium --confidence-level high
+  allow_failure: false
+  artifacts:
+    paths:
+      - bandit-report.json
+    expire_in: 30 days
+
+# ============================================================================
+# QUALITY STAGE: Code quality metrics
+# ============================================================================
+
+code-quality:
+  stage: security
+  <<: *uv_sync
+  script:
+    - echo "Generating code quality report..."
+    - echo "=== Cyclomatic Complexity ==="
+    - uv run radon cc sfacts/ -a -s
+    - echo "=== Maintainability Index ==="
+    - uv run radon mi sfacts/ -s
+  allow_failure: true
+  artifacts:
+    paths:
+      - htmlcov/
+    expire_in: 30 days
+
+# ============================================================================
+# DEPLOY STAGE: Publish to PyPI via Trusted Publisher (OIDC)
+# ============================================================================
+
+publish_pypi:
+  stage: deploy
+  image: python:3.11
+  environment: release  # Must match the Environment defined in GitLab & PyPI
+  cache: []  # Do not restore uv cache / .venv into build context
+  id_tokens:
+    PYPI_ID_TOKEN:
+      aud: pypi
+  script:
+    - rm -rf .uv-cache .venv dist build
+    - pip install uv
+    - uv build
+    - uv publish --trusted-publishing always
+  rules:
+    - if: $CI_COMMIT_TAG

sfacts-2.3.0/CHANGELOG.md

@@ -0,0 +1,208 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2.3.0] — 2026-04-13
+
+### Fixed
+
+- **VRF assignment bug in bulk sync** (`sfacts/core/netbox/bulk_sync.py`)
+  - `_collect_all_objects` was traversing the wrong nesting level (`vrf_info["interfaces"]`
+    instead of `vrf_info["interfaces"]["interface"]`), causing every interface to fall back
+    to the `default` VRF regardless of actual assignment in NAPALM facts
+  - Also limited the search to VLAN interfaces only; now all interface types are checked
+  - Non-default VRFs now correctly take priority over `default`/`global`/`master`
+- **VRF assignment bug in standard sync** (`sfacts/core/netbox/vrf_sync.py`)
+  - `_get_interface_vrf_from_facts` and `_sync_device_vrf_assignments` updated with the same
+    non-default priority logic
+- **`purge-netbox` did not remove device types and device roles**
+  - Full purge now deletes `device_types` and `device_roles` after devices, respecting FK
+    constraints
+- **Discovery drops devices with unrecognised device types**
+  - `SSHDetect.autodetect()` returning `None` (e.g. Ruijie SGE300) previously caused the
+    device to be silently discarded
+  - Now falls back to `device_type=generic` and still extracts hostname, keeping the device
+    in the discovery artifact
+- **ANSI escape sequences in hostname** (`sfacts/core/scanner.py`)
+  - `find_prompt()` output could contain ANSI codes (e.g. `\x1b[K`) that polluted the
+    extracted hostname; stripped with a regex before storing
+- **`collect-facts` error on `generic` devices** (`sfacts/core/collector.py`)
+  - Previously raised `Unsupported device type: generic`, failing the entire flow run
+  - Now returns partial facts (hostname, empty interfaces/IPs/VRFs) with
+    `connection_successful=True` and a descriptive note; pipeline continues
+
+### Added
+
+- **`facts-list` table artifact in `collect-facts`**
+  - Mirrors the `devices-list` table from `discover-network`
+  - Columns: Hostname, IP, Device Type, Status (✅/❌), Notes (model+version or error)
+  - `facts-summary` markdown also updated to list per-device hostname, IP, type, status, notes
+- **VRF & interface test coverage** (`tests/unit/`)
+  - `TestVrfSyncMixin` (8 tests) — covers `_get_interface_vrf_from_facts` and
+    `_sync_device_vrf_assignments` including the overlap regression case (interface in both
+    default and explicit VRF → explicit wins)
+  - `TestBulkVrfCollection` (6 tests) — covers `_collect_all_objects` bulk path for Vlan,
+    Ethernet, fallback-to-default, overlap priority, VRF name collection, fixture end-to-end
+  - `conftest.py` `sample_facts_data` VRF structure updated to correct NAPALM
+    `interfaces.interface` nested format; `switch01` gets a `data` VRF for `Vlan10`
+- **Blueprint: generic device re-identification**
+  - `docs/blueprint-generic-device-reidentification.md` — full design for optional secondary
+    probe-based re-identification of `generic` devices, NAPALM handoff, and Netmiko-only
+    collector fallback
+
+### Changed
+
+- **NetBox sync code split into focused modules** (`sfacts/core/netbox/` package)
+  - `sfacts/core/netbox_sync.py` and `sfacts/core/netbox_bulk_sync.py` (God classes, ~1500
+    and ~1015 lines) replaced by a structured package:
+    - `client.py` — `NetBoxClient`, connection management
+    - `base_objects.py` — `BaseObjectsMixin`, sites / roles / manufacturers / device types
+    - `interface_sync.py` — `InterfaceSyncMixin`
+    - `vrf_sync.py` — `VrfSyncMixin`
+    - `ipam_sync.py` — `IpamSyncMixin`, VLANs / prefixes / IPs
+    - `mac_sync.py` — `MacSyncMixin`
+    - `device_sync.py` — `NetBoxSyncManager` (orchestrator, inherits all mixins)
+    - `bulk_helpers.py` — `BulkHelpersMixin`, adaptive batching
+    - `bulk_sync.py` — `NetBoxBulkSyncManager`
+  - Old module paths re-exported for backward compatibility then removed
+
+---
+
+## [2.2.0] — 2026-04-12
+
+### Fixed
+
+- **Resolved all ruff/mypy lint errors** across new Prefect files (flows, tasks, deployments)
+  - Fixed E501, F841, B007, B904, SIM102, E722, S112, C401, W291 violations
+  - Applied ruff auto-fixes and unsafe-fixes for auto-fixable issues
+  - Added `type: ignore` for Prefect library false positives
+  - Updated mypy config: `python_version` 3.8 → 3.9, exclude `.venv`, skip `docket`
+  - Added `S107`/`S110`/`S112` to ruff per-file-ignores for flows and tests
+- **Fixed `__init__.py` version** from `1.0.0` to `2.1.0` to match `pyproject.toml`
+- **NetBox bulk sync now handles existing manufacturers, device roles, and device types**
+  - Fixed issue where bulk sync would create 0 devices when objects already existed in NetBox
+  - Added fallback logic to fetch existing manufacturers, device roles, and device types from NetBox
+  - Gracefully handles 400 Bad Request errors for duplicate objects during bulk creation
+  - Ensures cache is properly populated with existing objects for successful device creation
+  - Tested with 5 devices: successfully creates devices, interfaces, IPs, and MACs
+
+### Added
+
+- **Modular Prefect deployments with automatic artifact fetching**
+  - `sfacts/flows/modular_flows.py` — 3 independent deployments replacing 4 monolithic ones:
+    - `discover-network` — Auto-detects subnet vs tunnel mode from target format
+    - `collect-facts` — Gathers device facts with auto-fetch from latest discovery
+    - `sync-to-netbox` — Syncs to NetBox with auto-fetch from latest facts
+  - Automatic artifact retrieval using Prefect API:
+    - Auto-fetches latest artifacts by key (`devices-json`, `facts-json`)
+    - Optional `flow_run_id` parameter for specific artifact selection
+    - Proper Prefect filter objects for reliable artifact queries
+  - Enhanced artifact creation for all deployments:
+    - Discovery: summary markdown with hostnames, device table, JSON for next step
+    - Facts: summary markdown with hostnames and software versions, full JSON for NetBox sync
+    - NetBox sync: results summary with statistics
+  - `serve.py` — Deployment server script for running all flows
+  - `docs/prefect-modular-workflows.md` — Complete workflow documentation
+  - Benefits:
+    - Zero manual JSON copying between deployments
+    - Flexible workflows: run steps independently or skip discovery
+    - Clean separation of concerns
+    - Data persistence via Prefect artifacts
+    - Beautiful UI visualization with tables and markdown
+
+- **GitLab CI/CD pipeline** (`.gitlab-ci.yml`, `.gitlab-ci-simple.yml`)
+  - Four stages: lint, test, security, build
+  - Ruff linting and format checking, MyPy type checking
+  - Unit tests with coverage reporting (Cobertura XML + HTML)
+  - Dependency vulnerability scanning (Safety), Bandit security analysis
+  - Code quality metrics (Radon cyclomatic complexity, maintainability index)
+  - Python package building on main branch and tags
+  - Added `bandit>=1.7.10` to dev-dependencies
+  - Local CI runner script: `scripts/test-ci-locally.sh`
+  - GitLab merge request template: `.gitlab/merge_request_templates/default.md`
+  - Documentation: `docs/CI-CD-SETUP.md`
+
+- **Hostname extraction during discovery**
+  - `sfacts/core/scanner.py` — Enhanced device identification to extract hostnames
+  - Uses Netmiko's `base_prompt` attribute for robust hostname extraction
+  - Handles various prompt formats: `user@hostname`, `hostname/vsys1`, `hostname#`, etc.
+  - Hostnames available immediately in discovery artifacts (no need to wait for fact collection)
+  - Falls back to IP-based or port-based names if extraction fails
+
+- **Prefect Secret block support for credentials**
+  - Secure credential storage using Prefect Secret blocks
+  - Separate blocks for username (`lab-username`) and password (`lab-password`)
+  - Automatic credential loading from Secret blocks (default behavior)
+  - Manual credential override option via `use_secret_block=False`
+  - No hardcoded credentials in deployment parameters
+  - Credentials not visible in Prefect UI or logs
+  - Easy credential rotation without code changes
+
+- **Prefect workflow integration** (`sfacts[workflows]` optional extra)
+  - `sfacts/tasks/` — Prefect tasks wrapping the core sfacts API:
+    - `discover_devices` — subnet scan with automatic credential injection and retries
+    - `discover_tunnel_devices` — SSH tunnel scan (containerlab / remote labs)
+    - `collect_facts` — parallel NAPALM fact collection with retry logic
+    - `sync_to_netbox` — NetBox IPAM sync (standard and bulk modes) with retries
+    - `test_netbox_connection` — NetBox connectivity check
+    - `export_nornir_inventory` — Nornir inventory YAML export
+  - `sfacts/flows/` — Legacy flows (maintained for compatibility):
+    - `full_inventory_flow` — complete pipeline: discover → collect → optional export → optional NetBox sync
+    - `tunnel_inventory_flow` — same pipeline over SSH tunnel
+    - `discovery_flow` — lightweight subnet scan (no fact collection)
+    - `tunnel_discovery_flow` — lightweight scan over SSH tunnel
+  - Discovery results are credential-safe when saved to disk (credentials stripped before write)
+  - Unit tests for all tasks and flows (`tests/unit/test_tasks/`, `tests/unit/test_flows/`)
+  - Documentation: `docs/prefect-integration.md`, `docs/prefect-modular-workflows.md`
+
+### Changed
+
+- **Prefect deployment architecture**:
+  - Default `export_inventory=False` for Prefect deployments (artifacts handle persistence)
+  - All artifact creation functions are async and properly awaited
+  - Deployments now use artifact-based data pipeline instead of in-memory state
+  - Credentials now use Prefect Secret blocks instead of dict parameters
+  - Device keys in fact collection use hostnames instead of `host:port` format
+- **Discovery improvements**:
+  - Scanner now extracts hostnames during device identification
+  - Artifacts display actual device hostnames instead of IP addresses
+- **Fact collection improvements**:
+  - Summary artifacts now show software versions from NAPALM `get_facts`
+  - Uses `software_version` field instead of deprecated `version` field
+- `pyproject.toml`: added `workflows` optional dependency group (`prefect>=3.0.0`)
+- `pyproject.toml`: added `prefect>=3.0.0` to dev dependencies for test execution
+- `pyproject.toml`: extended per-file ruff ignores for tests (`S105`, `S106`) to cover
+  credential fixtures used in unit tests
+
+---
+
+## [2.1.0] — 2025-09-11
+
+### Added
+
+- Comprehensive test suite (unit, integration, e2e, performance)
+- Ruff code-quality improvements across the entire codebase
+
+## [2.0.0]
+
+### Added
+
+- NetBox bulk synchronization (`NetBoxBulkSyncManager`) with 10-50x performance improvement
+- `sfacts netbox sync --bulk` CLI flag
+- `sfacts netbox validate` subcommand
+- Primary MAC address assignment (`primary_mac_assignment.py`)
+
+## [1.0.0]
+
+### Added
+
+- Initial release
+- Network device discovery via subnet scanning (`SubnetScanner`, `NetworkDiscovery`)
+- Device fact collection via NAPALM (`BasicFactCollector`)
+- NetBox IPAM synchronization (`NetBoxSyncManager`)
+- Nornir inventory export (`NornirInventoryExporter`)
+- Secure credential management (`CredentialManager`)
+- CLI: `sfacts discover`, `sfacts facts`, `sfacts export`, `sfacts netbox`

sfacts-2.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Netodata Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.