netbox-data-import 2.3.0__tar.gz → 2.5.0__tar.gz

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: netbox-data-import
-Version: 2.3.0
+Version: 2.5.0
 Summary: NetBox plugin for importing data from external DCIM systems
 Author: Marcin Zieba
 License-Expression: Apache-2.0
@@ -39,14 +39,14 @@ A NetBox plugin for importing device inventory and rack layouts from external DC
 
 This plugin allows you to:
 
-- Define **import profiles** that configure how your source data (Excel/CSV) maps to NetBox objects
+- Define **import profiles** that configure how your source data (Excel) maps to NetBox objects
 - Configure **field mappings** per column with transformation rules
 - Define **class/role mappings** to translate your source system's device categories to NetBox roles
 - Synchronize a mapped **primary contact** into native NetBox Contacts and Contact Assignments
 - **Preview** the import before applying changes
 - **Run imports in the background**, track processed rows, and resume the progress page
 
-Currently supports importing from Excel (`.xlsx`) files in the CANS format, with the mapping configuration stored in NetBox so it can be reused and adapted for other source formats.
+Currently supports importing from Excel (`.xlsx`) workbooks with a flat, one-row-per-device layout. The mapping configuration is stored in NetBox, so it can be reused and adapted for other source formats.
 
 ## Requirements
 

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/README.md

@@ -16,14 +16,14 @@ A NetBox plugin for importing device inventory and rack layouts from external DC
 
 This plugin allows you to:
 
-- Define **import profiles** that configure how your source data (Excel/CSV) maps to NetBox objects
+- Define **import profiles** that configure how your source data (Excel) maps to NetBox objects
 - Configure **field mappings** per column with transformation rules
 - Define **class/role mappings** to translate your source system's device categories to NetBox roles
 - Synchronize a mapped **primary contact** into native NetBox Contacts and Contact Assignments
 - **Preview** the import before applying changes
 - **Run imports in the background**, track processed rows, and resume the progress page
 
-Currently supports importing from Excel (`.xlsx`) files in the CANS format, with the mapping configuration stored in NetBox so it can be reused and adapted for other source formats.
+Currently supports importing from Excel (`.xlsx`) workbooks with a flat, one-row-per-device layout. The mapping configuration is stored in NetBox, so it can be reused and adapted for other source formats.
 
 ## Requirements
 

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/__init__.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from netbox.plugins import PluginConfig
 
-__version__ = "2.3.0"
+__version__ = "2.5.0"
 
 
 class NetBoxDataImportConfig(PluginConfig):

netbox_data_import-2.5.0/netbox_data_import/_dns_worker.py

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2026 Marcin Zieba <marcinpsk@gmail.com>
+"""Resolve one host with libc in a deadline-bounded child process."""
+
+import json
+import signal
+import socket
+import sys
+
+from typing import TextIO
+
+
+def main(stdin: TextIO, stdout: TextIO) -> int:
+    """Read one request from stdin and write its ordered addresses to stdout."""
+    try:
+        host, port, timeout = json.load(stdin)
+    except Exception:  # noqa: BLE001 - the parent exposes one typed resolution failure
+        return 1
+    # The alarm is the deadline. An unarmed worker must crash, not read as a failed lookup.
+    signal.setitimer(signal.ITIMER_REAL, timeout)
+    try:
+        answers = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
+        addresses = list(dict.fromkeys(str(answer[4][0]) for answer in answers))
+    except Exception:  # noqa: BLE001 - the parent exposes one typed resolution failure
+        return 1
+    json.dump(addresses, stdout)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - child entry point; main() is tested in process
+    raise SystemExit(main(sys.stdin, sys.stdout))

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/adapter_forms.py

@@ -91,7 +91,7 @@ class FlatWorkbookConfigForm(AdapterConfigForm):
         max_length=100,
         required=False,
         initial="",
-        help_text="NetBox custom field name to store the source ID in (e.g. 'cans_id')",
+        help_text="NetBox custom field name to store the source ID in (e.g. 'external_id')",
     )
     update_existing = forms.BooleanField(
         required=False,

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/adapters.py

@@ -13,13 +13,14 @@ the NetBox boundary.
 
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from typing import TYPE_CHECKING
 
 from .catalog import OutputKind, has_implemented_module
+from .values import identity_text
 
 if TYPE_CHECKING:
-    from .trace_workbook import SourceTrace
+    from .source_trace import SourceTrace
 
 
 class UnknownSourceAdapter(Exception):
@@ -39,6 +40,16 @@ class SourceDiagnostic:
     row_number: int | None = None
 
 
+def _foreign_row_number(row) -> int | None:
+    """Return the source row a foreign row states, so its diagnostic can name a location."""
+    provenance = getattr(row, "provenance", ())
+    if provenance:
+        return provenance[0].row_start
+    if isinstance(row, dict):
+        return row.get("_row_number")
+    return None
+
+
 @dataclass(frozen=True)
 class SourceBatch:
     """The typed source items and source diagnostics from one file (section 1)."""
@@ -48,6 +59,46 @@ class SourceBatch:
     diagnostics: tuple[SourceDiagnostic, ...] = ()
     unused_columns: dict[str, dict] = field(default_factory=dict)
 
+    def __post_init__(self):
+        """Exclude a row the output kinds cannot carry, and mark a Trace with empty Device references."""
+        from .source_trace import SourceTrace
+
+        trace_batch = OutputKind.SOURCE_TRACE in self.output_kinds
+        expected = SourceTrace if trace_batch else dict
+        carries = "Source Traces" if trace_batch else "source rows"
+        rows = []
+        diagnostics = list(self.diagnostics)
+        for row in self.rows:
+            if not isinstance(row, expected):
+                # Excluded here, because every reader below reads the row the output kinds declare.
+                diagnostics.append(
+                    SourceDiagnostic(
+                        code="source.row_type_unexpected",
+                        message=f"This batch carries {carries} only; got {type(row).__name__}.",
+                        row_number=_foreign_row_number(row),
+                    )
+                )
+                continue
+            if not trace_batch:
+                rows.append(row)
+                continue
+            summary = row.endpoint_summary
+            references = [summary.from_termination, summary.to_termination, *row.corroboration]
+            for segment in row.segments:
+                references.extend((segment.left, segment.right))
+            if all(identity_text(reference.device) for reference in references):
+                rows.append(row)
+                continue
+            diagnostic = SourceDiagnostic(
+                code="trace.device_required",
+                message="A Source Trace Termination Reference must name a Device.",
+                row_number=row.provenance[0].row_start,
+            )
+            diagnostics.append(diagnostic)
+            rows.append(replace(row, errors=(*row.errors, diagnostic)))
+        object.__setattr__(self, "rows", tuple(rows))
+        object.__setattr__(self, "diagnostics", tuple(diagnostics))
+
 
 class SourceAdapter:
     """Base class for a Source Adapter declaration."""

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/cable_target.py

@@ -36,6 +36,13 @@ from .field_keys import (
 from .object_permissions import enforce_saved_object_permission
 from .plan import Diagnostic, Disposition, PlannedChange, Severity, SynchronizationUnit
 from .target_runtime import DeletedObject, PreconditionFailed
+from .trace_device_resolution import (
+    STALE as DEVICE_STALE,
+    collect_trace_device_evidence,
+    resolve_trace_devices,
+    resolved_trace_device,
+    source_device_key,
+)
 from .values import identity_text, source_text
 
 CABLE_STATUS = "connected"
@@ -111,6 +118,7 @@ class _Termination:
     kind: str
     device_id: int
     display: str
+    topology_display: str
 
     @property
     def key(self) -> tuple[str, int]:
@@ -207,7 +215,9 @@ class _TraceAnalysis:
     segments: list = field(default_factory=list)
     proven: dict = field(default_factory=dict)
     policies: dict = field(default_factory=dict)
+    devices: dict = field(default_factory=dict)
     terminations: dict = field(default_factory=dict)
+    resolution_started: bool = False
     topology_read: bool = False
     logical_cable: Any = None
     retains_logical_cable: bool = False
@@ -303,38 +313,19 @@ def _source_record(trace, segment_index: int) -> dict:
     }
 
 
-def _visible_devices(netbox_reader, names, *, _lock_rows=False) -> dict[str, list]:
-    """Return the visible Devices at the import target, grouped by comparison name."""
-    from django.db.models import Q
-
-    wanted = {identity_text(name) for name in names if identity_text(name)}
-    if not wanted:
-        return {}
-    devices = netbox_reader.devices()
-    if netbox_reader.site is not None:
-        devices = devices.filter(site=netbox_reader.site)
-    if _lock_rows:
-        devices = devices.order_by("pk").select_for_update(of=("self",))
-    lookup = Q()
-    for name in sorted(wanted):
-        lookup |= Q(name__iexact=name)
-    grouped: dict[str, list] = {}
-    for device in devices.filter(lookup):
-        comparison = identity_text(device.name)
-        if comparison in wanted:
-            grouped.setdefault(comparison, []).append(device)
-    return grouped
-
-
-def resolved_device_for(field_key: str, netbox_reader, *, _lock_rows=False):
+def resolved_device_for(field_key: str, netbox_reader, *, profile, _lock_rows=False):
     """Return the one visible Device a termination field key names, or None when it names no single one.
 
     A proposal freezes this reference and revalidates it, so the retrieval and the freshness check
     have to agree on what "the resolved Device" means.
     """
     device_name = parse_termination_field_key(field_key)["device"]
-    devices = _visible_devices(netbox_reader, [device_name], _lock_rows=_lock_rows).get(identity_text(device_name), [])
-    return devices[0] if len(devices) == 1 else None
+    return resolved_trace_device(
+        profile=profile,
+        reader=netbox_reader,
+        source_label=device_name,
+        lock_rows=_lock_rows,
+    )
 
 
 _RESOLVED_DEVICE_UNSET = object()
@@ -361,7 +352,7 @@ def eligible_terminations(
     kind = parsed["kind"]
     accessor = _READER_ACCESSOR_BY_KIND[kind]
     device = (
-        resolved_device_for(field_key, netbox_reader, _lock_rows=_lock_rows)
+        resolved_device_for(field_key, netbox_reader, profile=profile, _lock_rows=_lock_rows)
         if _resolved_device is _RESOLVED_DEVICE_UNSET
         else _resolved_device
     )
@@ -430,6 +421,7 @@ class _CableBatch:
     """Plan every Source Trace in one batch, so an identical shared segment plans once."""
 
     def __init__(self, traces, profile, netbox_reader, *, lock_plan_references: bool = False):
+        traces = tuple(traces)
         self.profile = profile
         self.reader = netbox_reader
         self.actor = netbox_reader.actor
@@ -445,7 +437,19 @@ class _CableBatch:
         self._occupied: dict[tuple[str, int], _ExistingCable] = {}
         self._mapping_rows: dict[str, Any] | None = None
         self._stored = self._stored_resolutions()
-        self._devices = self._load_devices()
+        active_traces = (analysis.trace for analysis in self.analyses if not analysis.stopped)
+        self._device_evidence = collect_trace_device_evidence(active_traces)
+        self._device_resolutions = resolve_trace_devices(
+            profile=self.profile,
+            reader=self.reader,
+            evidence=self._device_evidence,
+            lock_rows=self.lock_plan_references,
+        )
+        self._device_displays = {
+            resolution.device.pk: str(resolution.device)
+            for resolution in self._device_resolutions.values()
+            if resolution.device is not None
+        }
         self._resolve_terminations()
         self._load_mappings()
         self._build_segments()
@@ -512,16 +516,6 @@ class _CableBatch:
         ).select_related("selected_object_type")
         return {row.field_key: row for row in rows}
 
-    def _load_devices(self) -> dict[str, list]:
-        """Return the visible Devices every planned trace names, grouped by comparison name."""
-        names = [
-            reference.device
-            for analysis in self.analyses
-            if not analysis.stopped
-            for reference in self._references(analysis.trace)
-        ]
-        return _visible_devices(self.reader, names)
-
     def _components_for(self, device_id: int, kind: str) -> dict[str, list]:
         """Return one Device's terminations of one kind, grouped by comparison name."""
         cached = self._components.get((device_id, kind))
@@ -537,6 +531,7 @@ class _CableBatch:
         for analysis in self.analyses:
             if analysis.stopped:
                 continue
+            analysis.resolution_started = True
             resolved: dict[tuple, _Termination] = {}
             for reference in self._references(analysis.trace):
                 if reference.identity_key in resolved:
@@ -548,19 +543,24 @@ class _CableBatch:
 
     def _resolve_one(self, analysis: _TraceAnalysis, reference) -> _Termination | None:
         """Return the NetBox object one Termination Reference names, or record the open decision."""
-        devices = self._devices.get(identity_text(reference.device), [])
-        if len(devices) != 1:
-            analysis.block("trace.device_unresolved", {**_reference_display(reference), "matches": len(devices)})
+        resolution = self._device_resolutions[source_device_key(reference.device)]
+        analysis.devices.setdefault(resolution.evidence.key, resolution.to_question())
+        if resolution.device is None:
+            code = "trace.device_resolution_stale" if resolution.state == DEVICE_STALE else "trace.device_unresolved"
+            analysis.block(
+                code,
+                {**_reference_display(reference), "matches": resolution.exact_match_count},
+            )
             # With no resolved Device there is nothing to pick from, so the picker cannot help here.
             self._record_resolution(
                 analysis,
                 reference,
                 UNRESOLVED,
                 None,
-                reason=f"The source names {len(devices)} matching Devices, so no port list applies.",
+                reason="Resolve the source Device before choosing one of its ports.",
             )
             return None
-        device = devices[0]
+        device = resolution.device
         stored = self._stored.get(_field_key(reference))
         if stored is not None:
             termination = self._stored_termination(analysis, reference, device, stored)
@@ -653,6 +653,7 @@ class _CableBatch:
             kind=_KIND_BY_MODEL_NAME[label.partition(".")[2]],
             device_id=component.device_id,
             display=str(component),
+            topology_display=f"{self._device_displays[component.device_id]} {component.name}",
         )
 
     def _load_mappings(self) -> None:
@@ -1197,8 +1198,8 @@ class _CableBatch:
         for index, stated in enumerate(stated_segments):
             planned = resolved.get(index)
             source_left, source_right = _endpoint_label(stated.left), _endpoint_label(stated.right)
-            left = source_left if planned is None else planned.left.display
-            right = source_right if planned is None else planned.right.display
+            left = source_left if planned is None else planned.left.topology_display
+            right = source_right if planned is None else planned.right.topology_display
             segments.append(
                 {
                     "index": index,
@@ -1222,9 +1223,11 @@ class _CableBatch:
             },
             "segments": segments,
             "logical_cable": self._logical_cable_display(analysis),
+            "resolution_started": analysis.resolution_started,
             "topology_known": analysis.topology_read,
             # A unit with no changes proposes nothing, so the panel must not offer to delete one.
             "deletes_logical_cable": writes and analysis.deleted_logical_cable is not None,
+            "devices": list(analysis.devices.values()),
             "terminations": list(analysis.terminations.values()),
         }
 

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/catalog.py

@@ -259,6 +259,7 @@ POLICY_SECTIONS: tuple[PolicySection, ...] = (
     PolicySection("source_resolutions", "Source Resolutions", _DEVICE_ONLY),
     PolicySection("device_existing_matches", "Device Existing Matches", _DEVICE_ONLY),
     PolicySection("ignored_field_differences", "Ignored Field Differences", _DEVICE_ONLY),
+    PolicySection("trace_device_resolutions", "Trace Device Resolutions", _TRACE_ONLY),
     PolicySection("termination_resolutions", "Termination Resolutions", _TRACE_ONLY),
     PolicySection("cable_class_mappings", "CableClass Mappings", _TRACE_ONLY),
 )

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/device_field_review.py

@@ -205,6 +205,7 @@ class DeviceFieldReview:
     differing: dict[str, dict[str, str]]
     ignored: dict[str, dict[str, str]]
     informational: dict[str, dict[str, str]]
+    matching: dict[str, dict[str, str]]
     effective_proposal: dict[str, Any]
     snapshots: dict[str, tuple[dict[str, str], dict[str, str]]]
 
@@ -272,7 +273,7 @@ class DeviceFieldReviewer:
         display_overrides: Mapping[str, str] | None = None,
     ):
         """Return the writable differences and the reported-only ones as two maps."""
-        differing, informational, _ = DeviceFieldReviewer._compare(
+        differing, informational, _, _ = DeviceFieldReviewer._compare(
             matched_device,
             proposal,
             display_overrides=display_overrides,
@@ -306,7 +307,7 @@ class DeviceFieldReviewer:
         display_overrides: Mapping[str, str] | None = None,
     ) -> DeviceFieldReview:
         """Return differing, ignored, and write-safe values for one Device."""
-        differing, informational, snapshots = self._compare(
+        differing, informational, matching, snapshots = self._compare(
             matched_device,
             proposal,
             display_overrides=display_overrides,
@@ -333,6 +334,7 @@ class DeviceFieldReviewer:
             differing=differing,
             ignored=ignored,
             informational=informational,
+            matching=matching,
             effective_proposal=effective,
             snapshots=snapshots,
         )
@@ -348,6 +350,7 @@ class DeviceFieldReviewer:
         display_overrides = display_overrides or {}
         differing: dict[str, dict[str, str]] = {}
         informational: dict[str, dict[str, str]] = {}
+        matching: dict[str, dict[str, str]] = {}
         snapshots: dict[str, tuple[dict[str, str], dict[str, str]]] = {}
         for definition in _FIELD_DEFINITIONS:
             if definition.target_field not in proposal or not definition.provided(proposal[definition.target_field]):
@@ -363,6 +366,11 @@ class DeviceFieldReviewer:
             if definition.target_field == "rack_name":
                 _scope_rack_canonical(netbox_snapshot, _device_rack_location_id(matched_device))
             if file_snapshot["canonical"] == netbox_snapshot["canonical"]:
+                # Recorded, not dropped: a sync has to be able to say what it leaves alone.
+                matching[definition.target_field] = {
+                    "netbox": netbox_snapshot["display"],
+                    "file": file_snapshot["display"],
+                }
                 continue
             snapshots[definition.target_field] = (file_snapshot, netbox_snapshot)
             values = {"netbox": netbox_snapshot["display"], "file": file_snapshot["display"]}
@@ -370,4 +378,38 @@ class DeviceFieldReviewer:
                 differing[definition.target_field] = values
             else:
                 informational[definition.target_field] = values
-        return differing, informational, snapshots
+        return differing, informational, matching, snapshots
+
+
+#: What a sync would do to one field. The preview and the sync modal read these same names.
+SYNC_STATE_CHANGE = "change"
+SYNC_STATE_UNCHANGED = "unchanged"
+SYNC_STATE_IGNORED = "ignored"
+SYNC_STATE_NOT_WRITTEN = "not_written"
+
+#: The review bucket each state is built from, in the order the operator reads them.
+_SYNC_STATE_BUCKETS: tuple[tuple[str, str], ...] = (
+    (SYNC_STATE_CHANGE, "field_diff"),
+    (SYNC_STATE_UNCHANGED, "field_matching"),
+    (SYNC_STATE_IGNORED, "field_ignored"),
+    (SYNC_STATE_NOT_WRITTEN, "field_informational"),
+)
+
+
+def sync_change_preview(extra_data: Mapping[str, Any], labels: Mapping[str, str]) -> list[dict[str, str]]:
+    """Return one matched row's reviewed fields, each labelled with what a sync would do to it."""
+    entries = []
+    for state, bucket in _SYNC_STATE_BUCKETS:
+        values = extra_data.get(bucket) or {}
+        for target_field in sorted(values):
+            pair = values[target_field]
+            entries.append(
+                {
+                    "field": target_field,
+                    "label": labels.get(target_field) or target_field,
+                    "netbox": pair.get("netbox", ""),
+                    "file": pair.get("file", ""),
+                    "state": state,
+                }
+            )
+    return entries

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/forms.py

@@ -3,6 +3,7 @@
 import copy
 
 from django import forms
+from django.utils.html import format_html
 from dcim.models import Site, Location
 from tenancy.models import Tenant
 from netbox.forms import NetBoxModelBulkEditForm, NetBoxModelForm, NetBoxModelImportForm
@@ -25,6 +26,24 @@ from .models import (
 
 _EXPLICIT_NONE = "__explicit_none__"
 
+_CREDENTIAL_REFERENCE_EXAMPLE = """{
+  "backend": "vault_kv_v2",
+  "mount": "secret",
+  "path": "inference/backend",
+  "field": "api_key"
+}"""
+
+_CREDENTIAL_REFERENCE_HELP = format_html(
+    "This JSON tells the plugin where to find the API key in Vault. It does not contain the API key. "
+    "Configure Vault first in the NetBox plugin settings at "
+    "<code>PLUGINS_CONFIG[&quot;netbox_data_import&quot;][&quot;vault&quot;]</code>. "
+    '<details class="mt-1" data-credential-reference-help>'
+    "<summary>Show the required JSON shape</summary>"
+    '<pre class="mt-2 mb-0"><code>{}</code></pre>'
+    "</details>",
+    _CREDENTIAL_REFERENCE_EXAMPLE,
+)
+
 
 class _RuntimeCableChoiceField(forms.ChoiceField):
     """Render only current choices while letting shared validation classify submitted values."""
@@ -132,7 +151,7 @@ class ImportProfileForm(NetBoxModelForm):
             return self.instance.source_adapter
         if self.is_bound:
             return self.data.get(self.add_prefix("source_adapter")) or self.instance.source_adapter
-        return self.instance.source_adapter
+        return self.initial.get("source_adapter") or self.instance.source_adapter
 
     def clean(self):
         """Collect the adapter-declared fields into ``adapter_config``."""
@@ -306,8 +325,10 @@ class InferenceBackendForm(NetBoxModelForm):
             "tags",
         )
         help_texts = {
-            "credential_reference": (
-                "A typed Vault KV v2 reference: backend, mount, path and field. It never holds a key value."
+            "credential_reference": _CREDENTIAL_REFERENCE_HELP,
+            "model": (
+                "Enter the exact model id. After you save the backend, run the connection test to discover "
+                "model suggestions when the endpoint supports them."
             ),
         }
 

{netbox_data_import-2.3.0 → netbox_data_import-2.5.0}/netbox_data_import/import_engine.py

@@ -18,7 +18,7 @@ from django.db import DatabaseError
 from . import adapter_config, adapters, catalog, target_modules
 from .models import FailureReason, ImportExecution, SourceDocument, locked_profile_policy
 from .netbox_reader import NetBoxReader, PlanningTargetUnavailable
-from .object_permissions import ObjectPermissionDenied
+from .object_permissions import ObjectPermissionDenied, clear_user_permission_caches
 from .plan import Diagnostic, Disposition, ImportPlan, PlanInvalid, Severity, executable_units, merge_changes
 from .source_resolution import derive_effective_rows
 from .target_runtime import DeletedObject, ExecutionContext, PreconditionFailed
@@ -225,6 +225,7 @@ class ImportEngine:
         progress_callback,
     ) -> None:
         """Compare the selection against a fresh plan and apply it, inside the caller's transaction."""
+        clear_user_permission_caches(actor)
         current = cls.plan(
             profile,
             source_document,
@@ -253,6 +254,7 @@ class ImportEngine:
             if runtime is None:
                 raise EngineConfigurationError(f"No Target Module runtime is registered for '{change.target_module}'.")
             try:
+                clear_user_permission_caches(actor)
                 applied = runtime.apply(change, context)
             except (PreconditionFailed, ObjectPermissionDenied, ValidationError, DatabaseError) as exc:
                 raise _ExecutionFailed(