forward-netbox 2.2.0__tar.gz → 2.2.2__tar.gz

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forward-netbox
-Version: 2.2.0
+Version: 2.2.2
 Summary: NetBox plugin to sync Forward data into NetBox via built-in NQE queries
 License: MIT
 License-File: LICENSE
@@ -31,7 +31,9 @@ Forward 26.6 is the baseline for async NQE.
 
 | Plugin Release | NetBox Version | Status |
 | --- | --- | --- |
-| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
+| `v2.2.2` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix 504 gateway timeouts on large syncs: stop recomputing change-explainability on every poll during a long merge + back off poll to 15s |
+| `v2.2.1` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.2`; Add read-only forward_apply_identity_audit diagnostic to pinpoint 1-created/1-deleted idempotency churn |
+| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.1`; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
 | `v2.1.5` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.0`; Fix Prune orphans erroring on empty sites that still hold a VLAN/VM/prefix (delete only truly-empty sites) |
 | `v2.1.4` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.5`; Tag delete-eligible global IPAM (prefixes/VLANs/VRFs) for manual review |
 | `v2.1.3` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.4`; Prune empty orphan sites (zero devices + zero racks) alongside out-of-scope devices |

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/README.md

@@ -8,7 +8,9 @@ Forward 26.6 is the baseline for async NQE.
 
 | Plugin Release | NetBox Version | Status |
 | --- | --- | --- |
-| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
+| `v2.2.2` | `4.6.3` required; needs netbox-branching `1.1.0+` | Current release; Fix 504 gateway timeouts on large syncs: stop recomputing change-explainability on every poll during a long merge + back off poll to 15s |
+| `v2.2.1` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.2`; Add read-only forward_apply_identity_audit diagnostic to pinpoint 1-created/1-deleted idempotency churn |
+| `v2.2.0` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.1`; Fix devices mis-assigned the ACI platform; link Palo vsys / Fortinet vdom firewalls to their physical chassis |
 | `v2.1.5` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.2.0`; Fix Prune orphans erroring on empty sites that still hold a VLAN/VM/prefix (delete only truly-empty sites) |
 | `v2.1.4` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.5`; Tag delete-eligible global IPAM (prefixes/VLANs/VRFs) for manual review |
 | `v2.1.3` | `4.6.3` required; needs netbox-branching `1.1.0+` | Superseded by `v2.1.4`; Prune empty orphan sites (zero devices + zero racks) alongside out-of-scope devices |

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/forward_netbox/__init__.py

@@ -8,7 +8,7 @@ class NetboxForwardConfig(PluginConfig):
     name = "forward_netbox"
     verbose_name = "Forward Field Integration"
     description = "Sync Forward data into NetBox using built-in NQE queries."
-    version = "2.2.0"
+    version = "2.2.2"
     base_url = "forward"
     min_version = "4.6.3"
 

forward_netbox-2.2.2/forward_netbox/management/commands/forward_apply_identity_audit.py

@@ -0,0 +1,80 @@
+import json
+
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+
+from forward_netbox.models import ForwardSync
+from forward_netbox.utilities.apply_identity_audit import audit_apply_identity
+from forward_netbox.utilities.apply_identity_audit import SIMPLE_MODELS
+
+
+class Command(BaseCommand):
+    help = (
+        "Read-only diagnostic for the '1 created + 1 deleted every sync' churn. "
+        "For each simple-dimension model it compares Forward-computed natural keys "
+        "against NetBox-stored keys and reports the would-create / would-delete "
+        "leftovers. A model flagged churn_suspect (both non-empty, often 1/1) "
+        "names the object whose key Forward and NetBox disagree on. Never writes."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("--sync-id", type=int, default=0)
+        parser.add_argument("--sync-name", default="")
+        parser.add_argument(
+            "--models",
+            default="",
+            help=(
+                "Comma-separated subset of "
+                + ",".join(SIMPLE_MODELS)
+                + " (default: all enabled)."
+            ),
+        )
+        parser.add_argument("--limit", type=int, default=15, help="Sample size.")
+
+    def handle(self, *args, **options):
+        if options["sync_id"] and options["sync_name"]:
+            raise CommandError("Use either --sync-id or --sync-name, not both.")
+        sync = self._resolve_sync(options)
+        if sync is None:
+            raise CommandError("No sync found for the requested selector.")
+        if not sync.get_network_id():
+            raise CommandError("Sync source has no network configured.")
+
+        requested = [m.strip() for m in options["models"].split(",") if m.strip()]
+        unsupported = [m for m in requested if m not in SIMPLE_MODELS]
+        if unsupported:
+            raise CommandError(
+                "Unsupported --models: "
+                + ", ".join(unsupported)
+                + ". Supported: "
+                + ", ".join(SIMPLE_MODELS)
+                + "."
+            )
+
+        payload = audit_apply_identity(
+            sync,
+            models=requested or None,
+            sample_limit=int(options["limit"] or 15),
+        )
+        payload["remediation"] = (
+            ""
+            if not payload["churn_suspect_models"]
+            else (
+                "Churn suspected in: "
+                + ", ".join(payload["churn_suspect_models"])
+                + ". Compare each model's would_create_sample vs "
+                "would_delete_sample — they are the same object with a differing "
+                "key (e.g. a slug/name that Forward and NetBox compute "
+                "differently). That key difference is the bug to fix."
+            )
+        )
+        self.stdout.write(json.dumps(payload, indent=2, default=str))
+
+    def _resolve_sync(self, options):
+        sync_id = int(options.get("sync_id") or 0)
+        sync_name = (options.get("sync_name") or "").strip()
+        if sync_id:
+            return ForwardSync.objects.filter(pk=sync_id).first()
+        if sync_name:
+            return ForwardSync.objects.filter(name=sync_name).first()
+        return ForwardSync.objects.order_by("-id").first()

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/forward_netbox/templates/forward_netbox/forwardingestion.html

@@ -167,7 +167,7 @@
     <div id="ingestion_poll"
          class="col col-md-12"
          hx-get="{% url 'plugins:forward_netbox:forwardingestion_logs' pk=object.pk %}?stage={{ active_stage|default:'sync' }}"
-         hx-trigger="{% if job_running %}every 5s{% else %}none{% endif %}"
+         hx-trigger="{% if job_running %}every 15s{% else %}none{% endif %}"
          hx-target="#ingestion_logs"
          hx-select="#ingestion_logs"
          hx-select-oob="#ingestion_status:innerHTML,#ingestion_progress:innerHTML,#ingestion_statistics:innerHTML,#change_explainability:innerHTML,#object_tabs:innerHTML,#ingestion_merge_button:innerHTML"

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/forward_netbox/templates/forward_netbox/partials/ingestion_all.html

@@ -22,7 +22,7 @@
      hx-swap-oob="true"
      class="col col-md-12"
      hx-get="{{ logs_url }}?stage={{ active_stage }}"
-     hx-trigger="{% if polling_done %}none{% else %}every 5s{% endif %}"
+     hx-trigger="{% if polling_done %}none{% else %}every 15s{% endif %}"
      hx-target="#ingestion_logs"
      hx-select="#ingestion_logs"
      hx-select-oob="#ingestion_status:innerHTML,#ingestion_progress:innerHTML,#ingestion_statistics:innerHTML,#change_explainability:innerHTML,#object_tabs:innerHTML,#ingestion_merge_button:innerHTML"

forward_netbox-2.2.2/forward_netbox/tests/test_apply_identity_audit.py

@@ -0,0 +1,50 @@
+from types import SimpleNamespace
+
+from dcim.models import Site
+from django.test import TestCase
+
+from forward_netbox.utilities.apply_identity_audit import audit_apply_identity
+from forward_netbox.utilities.apply_identity_audit import audit_model_identity
+
+
+class ApplyIdentityAuditTest(TestCase):
+    def test_clean_when_keys_match(self):
+        Site.objects.create(name="Keep", slug="keep")
+        rows = [{"slug": "keep", "name": "Keep"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_create_count"], 0)
+        self.assertEqual(result["would_delete_count"], 0)
+        self.assertFalse(result["churn_suspect"])
+
+    def test_slug_mismatch_is_flagged_as_churn(self):
+        # Same logical site, but NetBox stored a different slug/name than Forward
+        # now computes -> the apply engine would delete the orphan and create the
+        # new one every sync. This is the 1-created/1-deleted signature.
+        Site.objects.create(name="Old Name", slug="old-slug")
+        rows = [{"slug": "new-slug", "name": "New Name"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_create_count"], 1)
+        self.assertEqual(result["would_delete_count"], 1)
+        self.assertTrue(result["churn_suspect"])
+        self.assertIn("old-slug", result["would_delete_sample"][0])
+        self.assertIn("new-slug", result["would_create_sample"][0])
+
+    def test_name_set_match_is_not_false_churn(self):
+        # Forward slug differs but the NAME still matches -> the apply engine
+        # matches on the name lookup_set, so it is NOT churn.
+        Site.objects.create(name="Shared", slug="stored-slug")
+        rows = [{"slug": "different-slug", "name": "Shared"}]
+        result = audit_model_identity("dcim.site", rows)
+        self.assertEqual(result["would_delete_count"], 0)
+        self.assertEqual(result["would_create_count"], 0)
+        self.assertFalse(result["churn_suspect"])
+
+    def test_audit_filters_to_enabled_models(self):
+        Site.objects.create(name="A", slug="a")
+        sync = SimpleNamespace(get_model_strings=lambda: ["dcim.site", "dcim.device"])
+        canned = {"dcim.site": [{"slug": "a", "name": "A"}]}
+        payload = audit_apply_identity(
+            sync, fetch_rows=lambda s, model: canned.get(model, [])
+        )
+        self.assertEqual(payload["models_audited"], ["dcim.site"])
+        self.assertEqual(payload["churn_suspect_models"], [])

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/forward_netbox/tests/test_log_export.py

@@ -338,6 +338,66 @@ class ForwardIngestionLogExportViewTest(TestCase):
         self.assertContains(response, "ipam.prefix 1")
         self.assertContains(response, "vrf 1")
 
+    def test_poll_defers_change_explainability_while_running(self):
+        # Regression (504 fix): while the job is running, the 5s/15s poll must
+        # NOT recompute change_explainability — doing so on every poll piles DB
+        # load onto the web workers during a long settling merge, which is what
+        # produces the gateway 504s on large platform-reclassification syncs.
+        running_sync = ForwardSync.objects.create(
+            name="sync-running-poll",
+            source=self.source,
+            parameters={"snapshot_id": "latestProcessed"},
+        )
+        running_ing = ForwardIngestion.objects.create(
+            sync=running_sync,
+            snapshot_selector="latestProcessed",
+            snapshot_id="snap-running",
+        )
+        now = timezone.now()
+        running_job = Job.objects.create(
+            object_type=ContentType.objects.get_for_model(ForwardIngestion),
+            object_id=running_ing.pk,
+            name="running-ingestion-job",
+            user=None,
+            status=JobStatusChoices.STATUS_RUNNING,
+            job_id="333e4567-e89b-12d3-a456-426614174002",
+            created=now,
+            started=now,
+            completed=None,
+            data={},
+        )
+        running_ing.job = running_job
+        running_ing.save(update_fields=["job"])
+
+        self.client.force_login(self.user)
+        with patch("forward_netbox.views.change_explainability_summary") as mock_ce:
+            response = self.client.get(
+                reverse(
+                    "plugins:forward_netbox:forwardingestion_logs",
+                    kwargs={"pk": running_ing.pk},
+                ),
+                headers={"HX-Request": "true"},
+            )
+        self.assertEqual(response.status_code, 200)
+        mock_ce.assert_not_called()
+
+    def test_poll_computes_change_explainability_when_done(self):
+        # The completed ingestion still computes it (the deferral is only while
+        # the job runs), so the operator sees the breakdown once it finishes.
+        self.client.force_login(self.user)
+        with patch(
+            "forward_netbox.views.change_explainability_summary",
+            return_value={"available": False},
+        ) as mock_ce:
+            self.client.get(
+                reverse(
+                    "plugins:forward_netbox:forwardingestion_logs",
+                    kwargs={"pk": self.ingestion.pk},
+                ),
+                headers={"HX-Request": "true"},
+            )
+        mock_ce.assert_called_once()
+
     def test_export_logs_compacts_large_execution_plan_items(self):
         sync = ForwardSync.objects.create(
             name="sync-log-export-plan-items",

forward_netbox-2.2.2/forward_netbox/utilities/apply_identity_audit.py

@@ -0,0 +1,173 @@
+# Read-only diagnostic for "1 created + 1 deleted every sync" idempotency churn.
+#
+# That pattern is an identity mismatch: one object whose natural key as Forward
+# computes it (the slug/name the query emits) does not equal its key as NetBox
+# stored it, so every sync the apply engine creates the Forward version and
+# deletes the NetBox orphan. Single-snapshot dup scans cannot see it (both keys
+# are individually unique); you have to compare the two key spaces.
+#
+# For each simple-dimension model this audits:
+#   * Forward computed keys  — from the emitted query rows.
+#   * NetBox stored keys     — from the live objects (lookup_key_from_object).
+# An object/row is matched if ANY of the model's lookup_sets matches (mirroring
+# the apply engine's multi-set lookup), so a slug change that still name-matches
+# is not a false positive. The leftovers are the churn:
+#   * would_create — Forward rows matched by no NetBox object.
+#   * would_delete — NetBox objects matched by no Forward row.
+# A model with would_create AND would_delete both small (often 1/1) names the
+# churning object and shows the differing key. NEVER writes anything.
+from .apply_engine_bulk import lookup_key_from_object
+from .apply_engine_bulk import lookup_key_from_values
+
+# Identity mirrors apply_engine_bulk's simple-model specs. The Forward query for
+# each model emits the lookup fields directly (slug/name/rd/vid/site_slug); we
+# read those off the row rather than recomputing slugify.
+SIMPLE_MODEL_IDENTITY = {
+    "dcim.site": {"lookup_sets": (("slug",), ("name",)), "row": ("slug", "name")},
+    "dcim.manufacturer": {
+        "lookup_sets": (("slug",), ("name",)),
+        "row": ("slug", "name"),
+    },
+    "dcim.devicerole": {
+        "lookup_sets": (("slug",), ("name",)),
+        "row": ("slug", "name"),
+    },
+    "dcim.platform": {"lookup_sets": (("slug",), ("name",)), "row": ("slug", "name")},
+    "dcim.devicetype": {"lookup_sets": (("slug",),), "row": ("slug",)},
+    "ipam.vrf": {"lookup_sets": (("rd",), ("name",)), "row": ("rd", "name")},
+}
+SIMPLE_MODELS = tuple(SIMPLE_MODEL_IDENTITY)
+
+
+def _model_class(model_string):
+    from dcim.models import DeviceRole
+    from dcim.models import DeviceType
+    from dcim.models import Manufacturer
+    from dcim.models import Platform
+    from dcim.models import Site
+    from ipam.models import VRF
+
+    return {
+        "dcim.site": Site,
+        "dcim.manufacturer": Manufacturer,
+        "dcim.devicerole": DeviceRole,
+        "dcim.platform": Platform,
+        "dcim.devicetype": DeviceType,
+        "ipam.vrf": VRF,
+    }[model_string]
+
+
+def audit_model_identity(model_string, rows, *, sample_limit=15):
+    """Compare Forward ``rows`` against NetBox objects for ``model_string`` and
+    report the would-create / would-delete leftovers (the churn candidates)."""
+    identity = SIMPLE_MODEL_IDENTITY[model_string]
+    lookup_sets = identity["lookup_sets"]
+    row_fields = identity["row"]
+    model = _model_class(model_string)
+
+    # Forward key sets (per lookup_set) from emitted row fields.
+    forward_keys = {lookup_set: set() for lookup_set in lookup_sets}
+    for row in rows:
+        values = {field: (row.get(field) or None) for field in row_fields}
+        for lookup_set in lookup_sets:
+            key = lookup_key_from_values(values, lookup_set, model_string=model_string)
+            if key is not None:
+                forward_keys[lookup_set].add(key)
+
+    netbox_keys = {lookup_set: set() for lookup_set in lookup_sets}
+    netbox_count = 0
+    would_delete = []
+    for obj in model.objects.all().iterator():
+        netbox_count += 1
+        obj_keys = {
+            lookup_set: lookup_key_from_object(
+                obj, lookup_set, model_string=model_string
+            )
+            for lookup_set in lookup_sets
+        }
+        for lookup_set, key in obj_keys.items():
+            if key is not None:
+                netbox_keys[lookup_set].add(key)
+        matched = any(
+            key is not None and key in forward_keys[lookup_set]
+            for lookup_set, key in obj_keys.items()
+        )
+        if not matched:
+            # Surface the stored key fields so the slug/name Forward and NetBox
+            # disagree on is visible next to the would_create label.
+            key_parts = ", ".join(
+                f"{field}={getattr(obj, field, None)}" for field in lookup_sets[0]
+            )
+            would_delete.append(f"{obj} ({key_parts})")
+
+    would_create = []
+    for row in rows:
+        values = {field: (row.get(field) or None) for field in row_fields}
+        matched = False
+        for lookup_set in lookup_sets:
+            key = lookup_key_from_values(values, lookup_set, model_string=model_string)
+            if key is not None and key in netbox_keys[lookup_set]:
+                matched = True
+                break
+        if not matched:
+            label = " / ".join(
+                str(values.get(field)) for field in row_fields if values.get(field)
+            )
+            would_create.append(label or "?")
+
+    return {
+        "model": model_string,
+        "forward_rows": len(rows),
+        "netbox_count": netbox_count,
+        "would_create_count": len(would_create),
+        "would_delete_count": len(would_delete),
+        "would_create_sample": sorted(would_create)[:sample_limit],
+        "would_delete_sample": sorted(would_delete)[:sample_limit],
+        # The churn signature: both sides non-empty and small => one logical
+        # object with two different keys (create the new, delete the orphan).
+        "churn_suspect": bool(would_create) and bool(would_delete),
+    }
+
+
+def audit_apply_identity(sync, *, models=None, sample_limit=15, fetch_rows=None):
+    """Audit each enabled simple-dimension model for create/delete key mismatches.
+
+    ``fetch_rows(sync, model_string) -> list[dict]`` is injectable for tests; the
+    default fetches via ``ForwardQueryFetcher``. Read-only.
+    """
+    enabled = set(sync.get_model_strings())
+    requested = models or SIMPLE_MODELS
+    selected = [m for m in requested if m in enabled and m in SIMPLE_MODEL_IDENTITY]
+    fetch_rows = fetch_rows or _default_fetch_rows
+
+    results = []
+    for model_string in selected:
+        rows = fetch_rows(sync, model_string)
+        results.append(
+            audit_model_identity(model_string, rows, sample_limit=sample_limit)
+        )
+    return {
+        "models_audited": selected,
+        "results": results,
+        "churn_suspect_models": [r["model"] for r in results if r["churn_suspect"]],
+    }
+
+
+def _default_fetch_rows(sync, model_string):
+    from .logging import SyncLogging
+    from .query_fetch import ForwardQueryFetcher
+
+    client = sync.source.get_client()
+    fetcher = ForwardQueryFetcher(sync, client, SyncLogging())
+    context = fetcher.resolve_context()
+    fetcher.run_preflight(context, model_strings=[model_string])
+    workloads = fetcher.fetch_workloads(
+        context,
+        model_strings=[model_string],
+        validate_rows=False,
+        include_diagnostics=False,
+    )
+    rows = []
+    for workload in workloads:
+        rows.extend(workload.upsert_rows or [])
+    return rows

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/forward_netbox/views.py

@@ -1374,19 +1374,27 @@ class ForwardIngestionLogView(LoginRequiredMixin, View):
         data["execution_state"] = _compact_execution_state_payload(
             json_safe_value(get_execution_display_state(ingestion.sync))
         )
-        data["change_explainability"] = change_explainability_summary(ingestion)
+        sync_running = ingestion.job and not ingestion.job.completed
+        merge_running = ingestion.merge_job and not ingestion.merge_job.completed
+        job_running = bool(sync_running or merge_running)
+        # Defer the change-explainability recompute while the job is running: it
+        # is only meaningful once staging/merge completes, and recomputing it on
+        # every poll piles DB load onto the web workers during a long settling
+        # merge (a large platform reclassification can run for minutes) — that
+        # contention is what produces the 504 gateway timeouts the customer sees.
+        data["change_explainability"] = (
+            {"available": False, "reason": "deferred_while_running"}
+            if job_running
+            else change_explainability_summary(ingestion)
+        )
         data["export_logs_url"] = reverse(
             "plugins:forward_netbox:forwardingestion_export_logs",
             kwargs={"pk": ingestion.pk},
         )
 
         if request.htmx:
-            sync_running = ingestion.job and not ingestion.job.completed
-            merge_running = ingestion.merge_job and not ingestion.merge_job.completed
             anything_ever_ran = ingestion.job or ingestion.merge_job
-            data["polling_done"] = (
-                bool(anything_ever_ran) and not sync_running and not merge_running
-            )
+            data["polling_done"] = bool(anything_ever_ran) and not job_running
         return render(request, self.template_name, data)
 
 
@@ -1447,7 +1455,13 @@ class ForwardIngestionView(generic.ObjectView):
         data["execution_state"] = _compact_execution_state_payload(
             json_safe_value(get_execution_display_state(instance.sync))
         )
-        data["change_explainability"] = change_explainability_summary(instance)
+        # Defer change-explainability while the job is running (see
+        # ForwardIngestionLogView): avoids recomputing it under merge contention.
+        data["change_explainability"] = (
+            {"available": False, "reason": "deferred_while_running"}
+            if data["job_running"]
+            else change_explainability_summary(instance)
+        )
         data["export_logs_url"] = reverse(
             "plugins:forward_netbox:forwardingestion_export_logs",
             kwargs={"pk": instance.pk},

{forward_netbox-2.2.0 → forward_netbox-2.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "forward-netbox"
-version = "2.2.0"
+version = "2.2.2"
 description = "NetBox plugin to sync Forward data into NetBox via built-in NQE queries"
 authors = ["Craig Johnson <craigjohnson@forwardnetworks.com>"]
 license = "MIT"