forward-netbox 2.2.0__tar.gz → 2.2.1__tar.gz

{forward_netbox-2.2.0 → forward_netbox-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forward-netbox
-Version: 2.2.0
+Version: 2.2.1
 Summary: NetBox plugin to sync Forward data into NetBox via built-in NQE queries
 License: MIT
 License-File: LICENSE
@@ -31,7 +31,8 @@ Forward 26.6 is the baseline for async NQE.
 
 | Plugin Release | NetBox Version | Status |
 | --- | --- | --- |
-| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
+| `v2.2.1` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Add read-only forward_apply_identity_audit diagnostic to pinpoint 1-created/1-deleted idempotency churn |
+| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.1`; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
 | `v2.1.5` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.0`; Fix Prune orphans erroring on empty sites that still hold a VLAN/VM/prefix (delete only truly-empty sites) |
 | `v2.1.4` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.5`; Tag delete-eligible global IPAM (prefixes/VLANs/VRFs) for manual review |
 | `v2.1.3` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.4`; Prune empty orphan sites (zero devices + zero racks) alongside out-of-scope devices |

{forward_netbox-2.2.0 → forward_netbox-2.2.1}/README.md

@@ -8,7 +8,8 @@ Forward 26.6 is the baseline for async NQE.
 
 | Plugin Release | NetBox Version | Status |
 | --- | --- | --- |
-| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
+| `v2.2.1` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Add read-only forward_apply_identity_audit diagnostic to pinpoint 1-created/1-deleted idempotency churn |
+| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.1`; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
 | `v2.1.5` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.0`; Fix Prune orphans erroring on empty sites that still hold a VLAN/VM/prefix (delete only truly-empty sites) |
 | `v2.1.4` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.5`; Tag delete-eligible global IPAM (prefixes/VLANs/VRFs) for manual review |
 | `v2.1.3` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.4`; Prune empty orphan sites (zero devices + zero racks) alongside out-of-scope devices |

{forward_netbox-2.2.0 → forward_netbox-2.2.1}/forward_netbox/__init__.py

@@ -8,7 +8,7 @@ class NetboxForwardConfig(PluginConfig):
     name = "forward_netbox"
     verbose_name = "Forward Field Integration"
     description = "Sync Forward data into NetBox using built-in NQE queries."
-    version = "2.2.0"
+    version = "2.2.1"
     base_url = "forward"
     min_version = "4.6.3"
 

forward_netbox-2.2.1/forward_netbox/management/commands/forward_apply_identity_audit.py

@@ -0,0 +1,80 @@
+import json
+
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+
+from forward_netbox.models import ForwardSync
+from forward_netbox.utilities.apply_identity_audit import audit_apply_identity
+from forward_netbox.utilities.apply_identity_audit import SIMPLE_MODELS
+
+
+class Command(BaseCommand):
+    help = (
+        "Read-only diagnostic for the '1 created + 1 deleted every sync' churn. "
+        "For each simple-dimension model it compares Forward-computed natural keys "
+        "against NetBox-stored keys and reports the would-create / would-delete "
+        "leftovers. A model flagged churn_suspect (both non-empty, often 1/1) "
+        "names the object whose key Forward and NetBox disagree on. Never writes."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("--sync-id", type=int, default=0)
+        parser.add_argument("--sync-name", default="")
+        parser.add_argument(
+            "--models",
+            default="",
+            help=(
+                "Comma-separated subset of "
+                + ",".join(SIMPLE_MODELS)
+                + " (default: all enabled)."
+            ),
+        )
+        parser.add_argument("--limit", type=int, default=15, help="Sample size.")
+
+    def handle(self, *args, **options):
+        if options["sync_id"] and options["sync_name"]:
+            raise CommandError("Use either --sync-id or --sync-name, not both.")
+        sync = self._resolve_sync(options)
+        if sync is None:
+            raise CommandError("No sync found for the requested selector.")
+        if not sync.get_network_id():
+            raise CommandError("Sync source has no network configured.")
+
+        requested = [m.strip() for m in options["models"].split(",") if m.strip()]
+        unsupported = [m for m in requested if m not in SIMPLE_MODELS]
+        if unsupported:
+            raise CommandError(
+                "Unsupported --models: "
+                + ", ".join(unsupported)
+                + ". Supported: "
+                + ", ".join(SIMPLE_MODELS)
+                + "."
+            )
+
+        payload = audit_apply_identity(
+            sync,
+            models=requested or None,
+            sample_limit=int(options["limit"] or 15),
+        )
+        payload["remediation"] = (
+            ""
+            if not payload["churn_suspect_models"]
+            else (
+                "Churn suspected in: "
+                + ", ".join(payload["churn_suspect_models"])
+                + ". Compare each model's would_create_sample vs "
+                "would_delete_sample — they are the same object with a differing "
+                "key (e.g. a slug/name that Forward and NetBox compute "
+                "differently). That key difference is the bug to fix."
+            )
+        )
+        self.stdout.write(json.dumps(payload, indent=2, default=str))
+
+    def _resolve_sync(self, options):
+        sync_id = int(options.get("sync_id") or 0)
+        sync_name = (options.get("sync_name") or "").strip()
+        if sync_id:
+            return ForwardSync.objects.filter(pk=sync_id).first()
+        if sync_name:
+            return ForwardSync.objects.filter(name=sync_name).first()
+        return ForwardSync.objects.order_by("-id").first()

forward_netbox-2.2.1/forward_netbox/tests/test_apply_identity_audit.py

@@ -0,0 +1,50 @@
+from types import SimpleNamespace
+
+from dcim.models import Site
+from django.test import TestCase
+
+from forward_netbox.utilities.apply_identity_audit import audit_apply_identity
+from forward_netbox.utilities.apply_identity_audit import audit_model_identity
+
+
+class ApplyIdentityAuditTest(TestCase):
+    def test_clean_when_keys_match(self):
+        Site.objects.create(name="Keep", slug="keep")
+        rows = [{"slug": "keep", "name": "Keep"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_create_count"], 0)
+        self.assertEqual(result["would_delete_count"], 0)
+        self.assertFalse(result["churn_suspect"])
+
+    def test_slug_mismatch_is_flagged_as_churn(self):
+        # Same logical site, but NetBox stored a different slug/name than Forward
+        # now computes -> the apply engine would delete the orphan and create the
+        # new one every sync. This is the 1-created/1-deleted signature.
+        Site.objects.create(name="Old Name", slug="old-slug")
+        rows = [{"slug": "new-slug", "name": "New Name"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_create_count"], 1)
+        self.assertEqual(result["would_delete_count"], 1)
+        self.assertTrue(result["churn_suspect"])
+        self.assertIn("old-slug", result["would_delete_sample"][0])
+        self.assertIn("new-slug", result["would_create_sample"][0])
+
+    def test_name_set_match_is_not_false_churn(self):
+        # Forward slug differs but the NAME still matches -> the apply engine
+        # matches on the name lookup_set, so it is NOT churn.
+        Site.objects.create(name="Shared", slug="stored-slug")
+        rows = [{"slug": "different-slug", "name": "Shared"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_delete_count"], 0)
+        self.assertEqual(result["would_create_count"], 0)
+        self.assertFalse(result["churn_suspect"])
+
+    def test_audit_filters_to_enabled_models(self):
+        Site.objects.create(name="A", slug="a")
+        sync = SimpleNamespace(get_model_strings=lambda: ["dcim.site", "dcim.device"])
+        canned = {"dcim.site": [{"slug": "a", "name": "A"}]}
+        payload = audit_apply_identity(
+            sync, fetch_rows=lambda s, model: canned.get(model, [])
+        )
+        self.assertEqual(payload["models_audited"], ["dcim.site"])
+        self.assertEqual(payload["churn_suspect_models"], [])

forward_netbox-2.2.1/forward_netbox/utilities/apply_identity_audit.py

@@ -0,0 +1,175 @@
+# Read-only diagnostic for "1 created + 1 deleted every sync" idempotency churn.
+#
+# That pattern is an identity mismatch: one object whose natural key as Forward
+# computes it (the slug/name the query emits) does not equal its key as NetBox
+# stored it, so every sync the apply engine creates the Forward version and
+# deletes the NetBox orphan. Single-snapshot dup scans cannot see it (both keys
+# are individually unique); you have to compare the two key spaces.
+#
+# For each simple-dimension model this audits:
+#   * Forward computed keys  — from the emitted query rows.
+#   * NetBox stored keys     — from the live objects (lookup_key_from_object).
+# An object/row is matched if ANY of the model's lookup_sets matches (mirroring
+# the apply engine's multi-set lookup), so a slug change that still name-matches
+# is not a false positive. The leftovers are the churn:
+#   * would_create — Forward rows matched by no NetBox object.
+#   * would_delete — NetBox objects matched by no Forward row.
+# A model with would_create AND would_delete both small (often 1/1) names the
+# churning object and shows the differing key. NEVER writes anything.
+from .apply_engine_bulk import lookup_key_from_object
+from .apply_engine_bulk import lookup_key_from_values
+
+# Identity mirrors apply_engine_bulk's simple-model specs. The Forward query for
+# each model emits the lookup fields directly (slug/name/rd/vid/site_slug); we
+# read those off the row rather than recomputing slugify.
+SIMPLE_MODEL_IDENTITY = {
+    "dcim.site": {"lookup_sets": (("slug",), ("name",)), "row": ("slug", "name")},
+    "dcim.manufacturer": {
+        "lookup_sets": (("slug",), ("name",)),
+        "row": ("slug", "name"),
+    },
+    "dcim.devicerole": {
+        "lookup_sets": (("slug",), ("name",)),
+        "row": ("slug", "name"),
+    },
+    "dcim.platform": {"lookup_sets": (("slug",), ("name",)), "row": ("slug", "name")},
+    "dcim.devicetype": {"lookup_sets": (("slug",),), "row": ("slug",)},
+    "ipam.vrf": {"lookup_sets": (("rd",), ("name",)), "row": ("rd", "name")},
+}
+SIMPLE_MODELS = tuple(SIMPLE_MODEL_IDENTITY)
+
+
+def _model_class(model_string):
+    from dcim.models import DeviceRole
+    from dcim.models import DeviceType
+    from dcim.models import Manufacturer
+    from dcim.models import Platform
+    from dcim.models import Site
+    from ipam.models import VRF
+
+    return {
+        "dcim.site": Site,
+        "dcim.manufacturer": Manufacturer,
+        "dcim.devicerole": DeviceRole,
+        "dcim.platform": Platform,
+        "dcim.devicetype": DeviceType,
+        "ipam.vrf": VRF,
+    }[model_string]
+
+
+def audit_model_identity(model_string, rows, *, sample_limit=15):
+    """Compare Forward ``rows`` against NetBox objects for ``model_string`` and
+    report the would-create / would-delete leftovers (the churn candidates)."""
+    identity = SIMPLE_MODEL_IDENTITY[model_string]
+    lookup_sets = identity["lookup_sets"]
+    row_fields = identity["row"]
+    model = _model_class(model_string)
+
+    # Forward key sets (per lookup_set) from emitted row fields.
+    forward_keys = {lookup_set: set() for lookup_set in lookup_sets}
+    forward_row_by_key = {}
+    for row in rows:
+        values = {field: (row.get(field) or None) for field in row_fields}
+        for lookup_set in lookup_sets:
+            key = lookup_key_from_values(values, lookup_set, model_string=model_string)
+            if key is not None:
+                forward_keys[lookup_set].add(key)
+                forward_row_by_key.setdefault((lookup_set, key), row)
+
+    netbox_keys = {lookup_set: set() for lookup_set in lookup_sets}
+    netbox_count = 0
+    would_delete = []
+    for obj in model.objects.all().iterator():
+        netbox_count += 1
+        obj_keys = {
+            lookup_set: lookup_key_from_object(
+                obj, lookup_set, model_string=model_string
+            )
+            for lookup_set in lookup_sets
+        }
+        for lookup_set, key in obj_keys.items():
+            if key is not None:
+                netbox_keys[lookup_set].add(key)
+        matched = any(
+            key is not None and key in forward_keys[lookup_set]
+            for lookup_set, key in obj_keys.items()
+        )
+        if not matched:
+            # Surface the stored key fields so the slug/name Forward and NetBox
+            # disagree on is visible next to the would_create label.
+            key_parts = ", ".join(
+                f"{field}={getattr(obj, field, None)}" for field in lookup_sets[0]
+            )
+            would_delete.append(f"{obj} ({key_parts})")
+
+    would_create = []
+    for row in rows:
+        values = {field: (row.get(field) or None) for field in row_fields}
+        matched = False
+        for lookup_set in lookup_sets:
+            key = lookup_key_from_values(values, lookup_set, model_string=model_string)
+            if key is not None and key in netbox_keys[lookup_set]:
+                matched = True
+                break
+        if not matched:
+            label = " / ".join(
+                str(values.get(field)) for field in row_fields if values.get(field)
+            )
+            would_create.append(label or "?")
+
+    return {
+        "model": model_string,
+        "forward_rows": len(rows),
+        "netbox_count": netbox_count,
+        "would_create_count": len(would_create),
+        "would_delete_count": len(would_delete),
+        "would_create_sample": sorted(would_create)[:sample_limit],
+        "would_delete_sample": sorted(would_delete)[:sample_limit],
+        # The churn signature: both sides non-empty and small => one logical
+        # object with two different keys (create the new, delete the orphan).
+        "churn_suspect": bool(would_create) and bool(would_delete),
+    }
+
+
+def audit_apply_identity(sync, *, models=None, sample_limit=15, fetch_rows=None):
+    """Audit each enabled simple-dimension model for create/delete key mismatches.
+
+    ``fetch_rows(sync, model_string) -> list[dict]`` is injectable for tests; the
+    default fetches via ``ForwardQueryFetcher``. Read-only.
+    """
+    enabled = set(sync.get_model_strings())
+    requested = models or SIMPLE_MODELS
+    selected = [m for m in requested if m in enabled and m in SIMPLE_MODEL_IDENTITY]
+    fetch_rows = fetch_rows or _default_fetch_rows
+
+    results = []
+    for model_string in selected:
+        rows = fetch_rows(sync, model_string)
+        results.append(
+            audit_model_identity(model_string, rows, sample_limit=sample_limit)
+        )
+    return {
+        "models_audited": selected,
+        "results": results,
+        "churn_suspect_models": [r["model"] for r in results if r["churn_suspect"]],
+    }
+
+
+def _default_fetch_rows(sync, model_string):
+    from .logging import SyncLogging
+    from .query_fetch import ForwardQueryFetcher
+
+    client = sync.source.get_client()
+    fetcher = ForwardQueryFetcher(sync, client, SyncLogging())
+    context = fetcher.resolve_context()
+    fetcher.run_preflight(context, model_strings=[model_string])
+    workloads = fetcher.fetch_workloads(
+        context,
+        model_strings=[model_string],
+        validate_rows=False,
+        include_diagnostics=False,
+    )
+    rows = []
+    for workload in workloads:
+        rows.extend(workload.upsert_rows or [])
+    return rows

{forward_netbox-2.2.0 → forward_netbox-2.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forward-netbox"
-version = "2.2.0"
+version = "2.2.1"
 description = "NetBox plugin to sync Forward data into NetBox via built-in NQE queries"
 authors = ["Craig Johnson <craigjohnson@forwardnetworks.com>"]
 license = "MIT"