ipfabric_netbox 4.3.2b9__py3-none-any.whl → 4.3.2b10__py3-none-any.whl

ipfabric_netbox/utilities/ipfutils.py

@@ -2,11 +2,8 @@ import json
 import logging
 from collections import Counter
 from copy import deepcopy
-from enum import Enum
 from functools import cache
-from functools import partial
 from importlib import metadata
-from typing import Any
 from typing import Callable
 from typing import TYPE_CHECKING
 from typing import TypeVar
@@ -14,8 +11,10 @@ from typing import TypeVar
 from core.exceptions import SyncError
 from core.signals import clear_events
 from dcim.models import Device
+from dcim.models import Site
 from dcim.models import VirtualChassis
 from dcim.signals import assign_virtualchassis_master
+from dcim.signals import sync_cached_scope_fields
 from django.conf import settings
 from django.core.exceptions import MultipleObjectsReturned
 from django.core.exceptions import ObjectDoesNotExist
@@ -136,26 +135,6 @@ class IPFabric(object):
             "user-agent"
         ] += f'; ipfabric-netbox/{metadata.version("ipfabric-netbox")}'  # noqa: E702
 
-    def get_snapshots(self) -> dict:
-        formatted_snapshots = {}
-        if self.ipf:
-            for snapshot_ref, snapshot in self.ipf.snapshots.items():
-                if snapshot.status != "done" and snapshot.finish_status != "done":
-                    continue
-                if snapshot_ref in ["$prev", "$lastLocked"]:
-                    continue
-                if snapshot.name:
-                    description = (
-                        snapshot.name
-                        + " - "
-                        + snapshot.end.strftime("%d-%b-%y %H:%M:%S")
-                    )
-                else:
-                    description = snapshot.end.strftime("%d-%b-%y %H:%M:%S")
-
-                formatted_snapshots[snapshot_ref] = (description, snapshot.snapshot_id)
-        return formatted_snapshots
-
     def get_table_data(self, table, device):
         filter = {"sn": ["eq", device.serial]}
         split = table.split(".")
@@ -194,15 +173,13 @@ class DataRecord:
 
     def __init__(
         self,
-        app: str,
-        model: str,
+        model_string: str,
         data: dict,
         # These values are filled later as the record is passed down the pipeline
         context: dict | None = None,
         transform_map: "IPFabricTransformMap | None" = None,
     ):
-        self.app = app
-        self.model = model
+        self.model_string = model_string
         self.data = data
         self.context = context or dict()
         self.transform_map = transform_map
@@ -212,8 +189,7 @@ class DataRecord:
             try:
                 self._hash = hash(
                     (
-                        self.app,
-                        self.model,
+                        self.model_string,
                         # Since the dicts are already ordered, it is safe to hash them
                         # .values() are mutable, this is fixed by tuple() to get same hash every time
                         make_hashable(self.data),
@@ -248,117 +224,6 @@ class DataRecord:
         self._hash = None  # Invalidate cached hash
 
 
-# TODO: 1) Store this as model in DB linked to Transform map
-# TODO: 2) Each table will be added to template data with customizable name (instead of just `object`)
-# TODO: and linked using common column (like `sn` for Device and Interface)
-# TODO: 3) Only pull required tables according to sync settings
-class Endpoints(Enum):
-    SITE = "inventory.sites"
-    DEVICE = "inventory.devices"
-    VIRTUALCHASSIS = "technology.platforms.stacks_members"
-    INTERFACE = "inventory.interfaces"
-    INVENTORYITEM = "inventory.pn"
-    VLAN = "technology.vlans.site_summary"
-    VRF = "technology.routing.vrf_detail"
-    PREFIX = "technology.managed_networks.networks"
-    IPADDRESS = "technology.addressing.managed_ip_ipv4"
-
-
-class Filters(Enum):
-    SITE = "site"
-    DEVICE = "device"
-    VIRTUALCHASSIS = "virtualchassis"
-    INTERFACE = "interface"
-    INVENTORYITEM = "inventoryitem"
-    VLAN = "vlan"
-    VRF = "vrf"
-    PREFIX = "prefix"
-    IPADDRESS = "ipaddress"
-
-    @staticmethod
-    def _site_filter(sites: dict | None) -> dict:
-        if sites:
-            return {"or": [{"siteName": ["eq", site]} for site in sites]}
-        return {}
-
-    @staticmethod
-    def _device_filter(sites: dict | None, child_table: bool = False) -> dict:
-        key = "vendor" if not child_table else "device.vendor"
-        excluded_vendors = ["aws", "azure"]
-        device_filter = {"and": [{key: ["neq", vendor]} for vendor in excluded_vendors]}
-        if sites:
-            site_filter = Filters._site_filter(sites)
-            device_filter["and"].append(site_filter)
-        return device_filter
-
-    @staticmethod
-    def _virtualchassis_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _interface_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _inventoryitem_filter(sites: dict | None) -> dict:
-        inventory_item_filter = {
-            "and": [
-                {"sn": ["empty", False]},
-                {"name": ["empty", False]},
-            ]
-        }
-        if site_filter := Filters._device_filter(sites, child_table=True):
-            inventory_item_filter["and"].append(site_filter)
-        return inventory_item_filter
-
-    @staticmethod
-    def _vlan_filter(sites: dict | None) -> dict:
-        # Remove VLANs with ID 0, minimum VLAN ID in NetBox is 1
-        vlan_filter = {"and": [{"vlanId": ["neq", 0]}]}
-        if site_filter := Filters._site_filter(sites):
-            vlan_filter["and"].append(site_filter)
-        return vlan_filter
-
-    @staticmethod
-    def _vrf_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _prefix_filter(sites: dict | None) -> dict:
-        if site_filter := Filters._device_filter(sites, child_table=True):
-            return {"and": [site_filter, {"and": [{"net": ["empty", False]}]}]}
-        else:
-            return {"and": [{"net": ["empty", False]}]}
-
-    @staticmethod
-    def _ipaddress_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def get_filter(endpoint: str, sites: dict | None) -> dict:
-        method_name = f"_{getattr(Filters, endpoint).value}_filter"
-        filter_func = getattr(Filters, method_name, None)
-        if filter_func:
-            return filter_func(sites)
-        return {}
-
-
-# TODO: Store also hierarchy of models (e.g. Device required Device and virtual chassis endpoints)
-
-
-class EndpointHandler:
-    def __init__(self, client: IPFClient, snapshot_id: str) -> None:
-        self.client = client
-        self.snapshot_id = snapshot_id
-
-    def resolve_endpoint(self, endpoint: str) -> Callable[..., Any]:
-        """Resolve a dot-separated endpoint string to the corresponding IPFClient attribute."""
-        obj = self.client
-        for attr in endpoint.split("."):
-            obj = getattr(obj, attr)
-        return partial(obj.all, snapshot_id=self.snapshot_id)
-
-
 def order_members(members: list[dict]) -> dict[str, list[dict]]:
     """Order VC members to dict, where key is master serial number and values are all members."""
     devices = {}
@@ -465,7 +330,7 @@ class IPFabricSyncRunner(object):
         exception: Exception,
         ingestion: "IPFabricIngestion",
         message: str = None,
-        model: str = None,
+        model_string: str = None,
         context: dict = None,
         data: dict = None,
     ) -> "tuple[bool, IPFabricIngestionIssue]":
@@ -476,7 +341,7 @@ class IPFabricSyncRunner(object):
 
         error_serial = self.get_error_serial(context, data)
         # Ignore models that do not have any dependencies by serial number
-        if error_serial and model not in ["ipaddress", "macaddress"]:
+        if error_serial and model_string not in ["ipam.ipaddress", "dcim.macaddress"]:
             self.error_serials.add(error_serial)
 
         # TODO: This is to prevent circular import issues, clean it up later.
@@ -487,7 +352,7 @@ class IPFabricSyncRunner(object):
                 ingestion=ingestion,
                 exception=exception.__class__.__name__,
                 message=message or getattr(exception, "message", str(exception)),
-                model=model,
+                model=model_string,
                 coalesce_fields={
                     k: v for k, v in context.items() if k not in ["defaults"]
                 },
@@ -515,19 +380,19 @@ class IPFabricSyncRunner(object):
                 # Logging section for logs inside job - facing user
                 self = args[0]
                 if isinstance(err, SearchError):
-                    if self.settings.get(err.model):
+                    if self.settings.get(err.model_string):
                         self.logger.log_failure(
-                            f"Aborting syncing `{err.model}` instance due to above error, please check your transform maps and/or existing data.",
+                            f"Aborting syncing `{err.model_string}` instance due to above error, please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                     else:
                         self.logger.log_failure(
-                            f"Syncing `{err.model}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
+                            f"Syncing `{err.model_string}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                 if isinstance(err, IPAddressDuplicateError):
                     self.logger.log_warning(
-                        f"IP Address `{err.data.get('address')}` already exists in `{err.model}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
+                        f"IP Address `{err.data.get('address')}` already exists in `{err.model_string}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
                         obj=self.sync,
                     )
                 else:
@@ -549,13 +414,11 @@ class IPFabricSyncRunner(object):
 
     def get_transform_context(self, record: DataRecord) -> DataRecord:
         if not record.transform_map:
-            raise SystemError(
-                f"No transform map available for {record.app}: {record.model}"
-            )
+            raise SystemError(f"No transform map available for {record.model_string}")
         try:
             record.context = record.transform_map.get_context(record.data)
         except Exception as err:
-            message = f"Error getting context for `{record.model}`."
+            message = f"Error getting context for `{record.model_string}`."
             if isinstance(err, ObjectDoesNotExist):
                 message += (
                     " Could not find related object using template in transform maps."
@@ -566,11 +429,14 @@ class IPFabricSyncRunner(object):
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 data=record.data,
             )
             raise SearchError(
-                message=message, data=record.data, model=record.model, issue_id=issue.pk
+                message=message,
+                data=record.data,
+                model_string=record.model_string,
+                issue_id=issue.pk,
             ) from err
 
         return record
@@ -581,67 +447,67 @@ class IPFabricSyncRunner(object):
         if error_serial and error_serial in self.error_serials:
             # We want to raise it as exception so it's shown in ingestion issues but can be filtered out.
             exception = RequiredDependencyFailedSkip(
-                message=f"Skipping syncing of `{record.model}` with serial `{error_serial}` due to previous errors.",
-                model=record.model,
+                message=f"Skipping syncing of `{record.model_string}` with serial `{error_serial}` due to previous errors.",
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             _, issue = self.create_or_get_sync_issue(
                 exception=exception,
                 ingestion=self.ingestion,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise exception
         record = self.get_transform_context(record)
         queryset = record.transform_map.target_model.model_class().objects
-        model_settings = self.settings.get(record.model, False)
+        model_settings = self.settings.get(f"{record.model_string}", False)
 
         obj = None
         try:
             connection_name = self.get_db_connection_name()
             if model_settings:
-                logger.info(f"Creating {record.model}")
+                logger.info(f"Creating {record.model_string}")
                 obj = record.transform_map.update_or_create_instance(
                     context=record.context,
                     tags=self.sync.tags.all(),
                     connection_name=connection_name,
                 )
             else:
-                logger.info(f"Getting {record.model}")
+                logger.info(f"Getting {record.model_string}")
                 record.context.pop("defaults", None)
                 obj = queryset.using(connection_name).get(**record.context)
         except queryset.model.DoesNotExist as err:
-            message = f"Instance of `{record.model}` not found."
+            message = f"Instance of `{record.model_string}` not found."
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
             ) from err
         except queryset.model.MultipleObjectsReturned as err:
-            message = f"Multiple instances of `{record.model}` found."
+            message = f"Multiple instances of `{record.model_string}` found."
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
@@ -650,12 +516,12 @@ class IPFabricSyncRunner(object):
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SyncDataError(
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
@@ -664,12 +530,16 @@ class IPFabricSyncRunner(object):
         return obj
 
     def collect_data(self):
+        # Importing here to avoid circular import
+        from ..models import IPFabricEndpoint
+
         try:
             self.logger.log_info(
                 "Collecting information from IP Fabric",
                 obj=self.sync.snapshot_data.source,
             )
             data = {}
+            # TODO: Replace endpoint value in SnapshotData with Endpoint link
             if self.sync.snapshot_data.source.type == IPFabricSourceTypeChoices.REMOTE:
                 # This requires data already pushed to NetBox by user, does not connect to IPF directly
                 self.logger.log_info(
@@ -679,34 +549,59 @@ class IPFabricSyncRunner(object):
                     raise SyncError(
                         "No snapshot data available. This is a remote sync. Push data to NetBox first."
                     )
-                for endpoint in Endpoints:
-                    data[endpoint.name.lower()] = list(
+                for endpoint in IPFabricEndpoint.objects.all():
+                    data[endpoint.endpoint] = list(
                         self.sync.snapshot_data.ipf_data.filter(
-                            type=endpoint.name.lower()
+                            type=endpoint.endpoint
                         ).values_list("data", flat=True)
                     )
             else:
                 # This pulls data directly from IP Fabric instance
+                # TODO: If certain columns are not enabled in IPF, sync will fail in odd ways
+                #  For example missing `nameOriginal` on Interface makes it unable to sync IPs
+                #  When pulling from endpoint, make sure to specify columns according to transform maps
+                #  This can be pulled using logic in IPFabricTransformMap.strip_source_data()
+                # TODO: Also cache it to improve performance, no need to regex every time!
                 self.logger.log_info(
                     "Local collector being used for snapshot data.", obj=self.sync
                 )
-                endpoint_handler = EndpointHandler(
-                    self.client,
-                    snapshot_id=self.settings["snapshot_id"],
-                )
-                ingestion_sites = self.settings.get("sites")
-                for endpoint in Endpoints:
-                    filters = Filters.get_filter(endpoint.name, ingestion_sites)
+                enabled_models = [
+                    param
+                    for param in self.sync.parameters.keys()
+                    if "." in param and self.sync.parameters.get(param) is True
+                ]
+                # Always pull devices for now since other models depend on them
+                enabled_models.append("dcim.device")
+                # Interfaces are required for IP Addresses assigned interface name
+                if "ipam.ipaddress" in enabled_models:
+                    enabled_models.append("dcim.interface")
+                transform_maps = self.sync.get_transform_maps()
+                for endpoint in IPFabricEndpoint.objects.all():
+                    tms_for_endpoint = transform_maps.filter(source_endpoint=endpoint)
+                    if not tms_for_endpoint.exists():
+                        raise SyncError(
+                            f"No transform map found for endpoint `{endpoint.endpoint}`."
+                        )
+                    if not [
+                        True
+                        for tm in tms_for_endpoint
+                        if ".".join(tm.target_model.natural_key()) in enabled_models
+                    ]:
+                        logger.debug(
+                            f"Skipping endpoint `{endpoint.endpoint}` as no enabled models are using it."
+                        )
+                        continue
+                    filters = endpoint.combine_filters(sync=self.sync)
                     logger.debug(
-                        f"Collecting data from endpoint: `{endpoint.value}` using filter `{json.dumps(filters)}`."
+                        f"Collecting data from endpoint: `{endpoint.endpoint}` using filter `{json.dumps(filters)}`."
                     )
-                    data[endpoint.name.lower()] = endpoint_handler.resolve_endpoint(
-                        endpoint.value
-                    )(
+                    data[endpoint.endpoint] = self.client.fetch_all(
+                        url=endpoint.endpoint,
+                        snapshot_id=self.settings["snapshot_id"],
                         filters=filters,
                     )
                     self.logger.log_info(
-                        f"Collected {len(data[endpoint.name.lower()])} items from endpoint `{endpoint.value}`.",
+                        f"Collected {len(data[endpoint.endpoint])} items from endpoint `{endpoint.endpoint}`.",
                         obj=self.sync.snapshot_data.source,
                     )
         except Exception as e:
@@ -723,60 +618,89 @@ class IPFabricSyncRunner(object):
             target_model__app_label=app, target_model__model=model
         )
 
+    def create_new_data_records(
+        self, app: str, model: str, data: list[dict] | dict[str, list[dict]]
+    ) -> set[DataRecord]:
+        """Create DataRecord objects for given app, model and data list if enabled in settings."""
+        if not self.sync.parameters.get(f"{app}.{model}"):
+            return set()
+        if isinstance(data, dict):
+            # Data is either already list or full data where we have to choose the list
+            transform_map = self.get_transform_map(app=app, model=model)
+            data = data.get(transform_map.source_endpoint.endpoint, [])
+        return set(
+            self.create_new_data_record(app=app, model=model, data=item)
+            for item in data
+        )
+
     def create_new_data_record(self, app: str, model: str, data: dict) -> DataRecord:
         """Extract only relevant source data according to transform map configuration."""
         transform_map = self.get_transform_map(app=app, model=model)
+        model_string = f"{app}.{model}"
         try:
             source_data = transform_map.strip_source_data(data)
         except KeyError as err:
             raise SyncError(
-                f"Missing key column {err.args[0]} in source data when preparing data for {app}_{model}."
+                f"Missing key column {err.args[0]} in source data when preparing data for {model_string}."
             ) from err
         return DataRecord(
-            app=app, model=model, data=source_data, transform_map=transform_map
+            model_string=model_string, data=source_data, transform_map=transform_map
         )
 
     def preprocess_data(self, data: dict) -> dict[str, set[DataRecord]]:
-        # TODO: Only process data according to settings to improve performance
         # Set those records that can't be iterated separately
         # Others are as empty set to define order which is shown in UI progress
         records = {
-            "site": set(
-                self.create_new_data_record(app="dcim", model="site", data=item)
-                for item in data.get("site", [])
+            "dcim.site": self.create_new_data_records(
+                app="dcim", model="site", data=data
             ),
-            "manufacturer": set(),
-            "devicetype": set(),
-            "platform": set(),
-            "devicerole": set(),
-            "device": set(),
-            "virtualchassis": set(),
-            "interface": set(),
-            "macaddress": set(),
-            "inventoryitem": set(
-                self.create_new_data_record(
-                    app="dcim", model="inventoryitem", data=item
-                )
-                for item in data.get("inventoryitem", [])
+            "dcim.manufacturer": set(),
+            "dcim.devicetype": set(),
+            "dcim.platform": set(),
+            "dcim.devicerole": set(),
+            "dcim.device": set(),
+            "dcim.virtualchassis": set(),
+            "dcim.interface": set(),
+            "dcim.macaddress": set(),
+            "dcim.inventoryitem": self.create_new_data_records(
+                app="dcim",
+                model="inventoryitem",
+                data=data,
             ),
-            "vlan": set(
-                self.create_new_data_record(app="ipam", model="vlan", data=item)
-                for item in data.get("vlan", [])
+            "ipam.vlan": self.create_new_data_records(
+                app="ipam",
+                model="vlan",
+                data=data,
             ),
-            "vrf": set(
-                self.create_new_data_record(app="ipam", model="vrf", data=item)
-                for item in data.get("vrf", [])
+            "ipam.vrf": self.create_new_data_records(
+                app="ipam",
+                model="vrf",
+                data=data,
             ),
-            "prefix": set(
-                self.create_new_data_record(app="ipam", model="prefix", data=item)
-                for item in data.get("prefix", [])
+            "ipam.prefix": self.create_new_data_records(
+                app="ipam",
+                model="prefix",
+                data=data,
             ),
-            "ipaddress": set(),
+            "ipam.ipaddress": set(),
         }
 
-        self.logger.log_info("Preparing devices", obj=self.sync)
-        members = order_members(data.get("virtualchassis", []))
-        devices, virtualchassis = prepare_devices(data.get("device", []), members)
+        if self.sync.parameters.get("dcim.virtualchassis"):
+            self.logger.log_info("Preparing virtual chassis members", obj=self.sync)
+            members = order_members(data.get("/technology/platforms/stack/members", []))
+        else:
+            members = []
+
+        if self.sync.parameters.get("dcim.device") or self.sync.parameters.get(
+            "dcim.virtualchassis"
+        ):
+            self.logger.log_info("Preparing devices", obj=self.sync)
+            devices, virtualchassis = prepare_devices(
+                data.get("/inventory/devices", []), members
+            )
+        else:
+            # We can skip iterating over devices since we don't need stack members for the rest of the models
+            devices, virtualchassis = data.get("/inventory/devices", []), []
 
         # We need to store primary IPs of Devices to assign them later
         # since they are not stored on Device object directly
@@ -784,31 +708,41 @@ class IPFabricSyncRunner(object):
         device_primary_ips = {}
 
         for device in devices:
-            records["manufacturer"].add(
-                self.create_new_data_record(
-                    app="dcim", model="manufacturer", data=device
+            if self.sync.parameters.get("dcim.manufacturer"):
+                records["dcim.manufacturer"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="manufacturer", data=device
+                    )
+                )
+            if self.sync.parameters.get("dcim.devicetype"):
+                records["dcim.devicetype"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="devicetype", data=device
+                    )
+                )
+            if self.sync.parameters.get("dcim.platform"):
+                records["dcim.platform"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="platform", data=device
+                    )
+                )
+            if self.sync.parameters.get("dcim.devicerole"):
+                records["dcim.devicerole"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="devicerole", data=device
+                    )
                 )
-            )
-            records["devicetype"].add(
-                self.create_new_data_record(app="dcim", model="devicetype", data=device)
-            )
-            records["platform"].add(
-                self.create_new_data_record(app="dcim", model="platform", data=device)
-            )
-            records["devicerole"].add(
-                self.create_new_data_record(app="dcim", model="devicerole", data=device)
-            )
             # This field is required by Device transform maps, but is set only when Device is part of VC.
             if "virtual_chassis" not in device:
                 device["virtual_chassis"] = None
-            records["device"].add(
-                self.create_new_data_record(app="dcim", model="device", data=device)
-            )
+            if self.sync.parameters.get("dcim.device"):
+                records["dcim.device"].add(
+                    self.create_new_data_record(app="dcim", model="device", data=device)
+                )
             device_primary_ips[device.get("sn")] = device.get("loginIp")
 
-        records["virtualchassis"] = set(
-            self.create_new_data_record(app="dcim", model="virtualchassis", data=item)
-            for item in virtualchassis
+        records["dcim.virtualchassis"] = self.create_new_data_records(
+            app="dcim", model="virtualchassis", data=virtualchassis
         )
 
         # `nameOriginal` is human-readable interface name hidden column in IP Fabric
@@ -826,53 +760,61 @@ class IPFabricSyncRunner(object):
             )
             raise SyncError(f"Error collecting source column name for interface: {e}")
 
-        self.logger.log_info("Preparing Interfaces", obj=self.sync)
         # Store human-readable interface names to use them later for IP Addresses
         readable_int_names = {}
-        for interface in data["interface"]:
-            interface_record = self.create_new_data_record(
-                app="dcim", model="interface", data=interface
-            )
-            interface_record.data["loginIp"] = device_primary_ips.get(
-                interface.get("sn")
-            )
-            records["interface"].add(interface_record)
-            readable_int_names[
-                f"{interface.get('sn')}_{interface.get('intName')}"
-            ] = interface.get(interface_key)
-            records["macaddress"].add(
-                self.create_new_data_record(
-                    app="dcim", model="macaddress", data=interface
-                )
-            )
+        if (
+            self.sync.parameters.get("dcim.interface")
+            or self.sync.parameters.get("dcim.macaddress")
+            or self.sync.parameters.get("ipam.ipaddress")
+        ):
+            self.logger.log_info("Preparing Interfaces", obj=self.sync)
+            for interface in data.get("/inventory/interfaces", []):
+                if self.sync.parameters.get("dcim.interface"):
+                    interface_record = self.create_new_data_record(
+                        app="dcim", model="interface", data=interface
+                    )
+                    interface_record.data["loginIp"] = device_primary_ips.get(
+                        interface.get("sn")
+                    )
+                    records["dcim.interface"].add(interface_record)
+                readable_int_names[
+                    f"{interface.get('sn')}_{interface.get('intName')}"
+                ] = interface.get(interface_key)
+                if self.sync.parameters.get("dcim.macaddress"):
+                    records["dcim.macaddress"].add(
+                        self.create_new_data_record(
+                            app="dcim", model="macaddress", data=interface
+                        )
+                    )
 
-        self.logger.log_info("Preparing IP Addresses", obj=self.sync)
-        for ip in data["ipaddress"]:
-            # We get `nameOriginal` from Interface table to get human-readable name instead fo `intName`
-            ip["nameOriginal"] = readable_int_names.get(
-                f"{ip.get('sn')}_{ip.get('intName')}"
-            )
-            # Let's skip IPs we cannot assign to an interface
-            if not ip["nameOriginal"]:
-                continue
-            ipaddress_record = self.create_new_data_record(
-                app="ipam", model="ipaddress", data=ip
-            )
-            # Store whether this IP is primary for the device
-            ipaddress_record.data["is_primary"] = ip.get(
-                "sn"
-            ) in device_primary_ips and device_primary_ips.get(
-                ip.get("sn")
-            ) == ipaddress_record.data.get(
-                "ip"
-            )
-            records["ipaddress"].add(ipaddress_record)
+        if self.sync.parameters.get("ipam.ipaddress"):
+            self.logger.log_info("Preparing IP Addresses", obj=self.sync)
+            for ip in data.get("/technology/addressing/managed-ip/ipv4", []):
+                # We get `nameOriginal` from Interface table to get human-readable name instead fo `intName`
+                ip["nameOriginal"] = readable_int_names.get(
+                    f"{ip.get('sn')}_{ip.get('intName')}"
+                )
+                # Let's skip IPs we cannot assign to an interface
+                if not ip["nameOriginal"]:
+                    continue
+                ipaddress_record = self.create_new_data_record(
+                    app="ipam", model="ipaddress", data=ip
+                )
+                # Store whether this IP is primary for the device
+                ipaddress_record.data["is_primary"] = ip.get(
+                    "sn"
+                ) in device_primary_ips and device_primary_ips.get(
+                    ip.get("sn")
+                ) == ipaddress_record.data.get(
+                    "ip"
+                )
+                records["ipam.ipaddress"].add(ipaddress_record)
 
-        for model, records_set in records.items():
-            if self.settings.get(model) and len(records_set):
-                self.logger.init_statistics(model, len(records_set))
+        for model_string, records_set in records.items():
+            if self.settings.get(model_string) and len(records_set):
+                self.logger.init_statistics(model_string, len(records_set))
                 self.logger.log_info(
-                    f"Prepared {len(records_set)} items for `{model}` to be synced.",
+                    f"Prepared {len(records_set)} items for `{model_string}` to be synced.",
                     obj=self.sync,
                 )
 
@@ -883,13 +825,8 @@ class IPFabricSyncRunner(object):
         self,
         record: DataRecord,
         stats: bool = True,
-        sync: bool = False,
     ) -> ModelTypeVar | None:
         """Sync a single item to NetBox."""
-        # The `sync` param is a workaround since we need to get some models (Device...) even when not syncing them.
-        if not sync:
-            return None
-
         if not record.data:
             return None
 
@@ -897,7 +834,7 @@ class IPFabricSyncRunner(object):
 
         # Only log when we successfully synced the item and asked for it
         if stats and instance:
-            self.logger.increment_statistics(model=record.model)
+            self.logger.increment_statistics(model_string=record.model_string)
 
         return instance
 
@@ -909,11 +846,7 @@ class IPFabricSyncRunner(object):
         stats: bool = True,
     ) -> ModelTypeVar | None:
         """Sync a single item to NetBox."""
-        synced_object = self.sync_model(
-            record=record,
-            sync=self.settings.get(record.model),
-            stats=stats,
-        )
+        synced_object = self.sync_model(record=record, stats=stats)
         if synced_object is None:
             return None
 
@@ -939,10 +872,10 @@ class IPFabricSyncRunner(object):
         if not items:
             return
 
-        app, model = (lambda x: (x.app, x.model))(next(iter(items)))
-        if not self.settings.get(model):
+        model_string = (lambda record: record.model_string)(next(iter(items)))
+        if not self.settings.get(model_string):
             self.logger.log_info(
-                f"Did not ask to sync {model}s, skipping.", obj=self.sync
+                f"Did not ask to sync {model_string}s, skipping.", obj=self.sync
             )
             return
 
@@ -958,7 +891,7 @@ class IPFabricSyncRunner(object):
         ingestion: "IPFabricIngestion" = None,
     ) -> None:
         """Sync devices separately to handle resetting primary IP."""
-        if not self.settings.get("device"):
+        if not self.settings.get("dcim.device"):
             self.logger.log_info(
                 "Did not ask to sync devices, skipping.", obj=self.sync
             )
@@ -989,13 +922,13 @@ class IPFabricSyncRunner(object):
                     exception=err,
                     ingestion=self.ingestion,
                     message="Error removing primary IP current device.",
-                    model=device.model,
+                    model_string=device.model_string,
                     data=device.data,
                 )
                 self.events_clearer.increment()
                 raise IPAddressPrimaryRemovalError(
                     data=device.data,
-                    model=device.model,
+                    model_string=device.model_string,
                     issue_id=issue.pk,
                 ) from err
             self.events_clearer.increment()
@@ -1032,13 +965,13 @@ class IPFabricSyncRunner(object):
                 exception=err,
                 ingestion=self.ingestion,
                 message="Error removing primary IP from other device.",
-                model=ip_address.model,
+                model_string=ip_address.model_string,
                 data=ip_address.data,
             )
             self.events_clearer.increment()
             raise IPAddressPrimaryRemovalError(
                 data=ip_address.data,
-                model=ip_address.model,
+                model_string=ip_address.model_string,
                 issue_id=issue.pk,
             ) from err
 
@@ -1063,13 +996,13 @@ class IPFabricSyncRunner(object):
                     exception=err,
                     ingestion=self.ingestion,
                     message="Error assigning primary IP to device.",
-                    model=ip_address.model,
+                    model_string=ip_address.model_string,
                     data=ip_address.data,
                 )
                 self.events_clearer.increment()
                 raise IPAddressPrimaryAssignmentError(
                     data=ip_address.data,
-                    model=ip_address.model,
+                    model_string=ip_address.model_string,
                     issue_id=issue.pk,
                 ) from err
         self.events_clearer.increment()
@@ -1086,7 +1019,7 @@ class IPFabricSyncRunner(object):
         Cleaning events queue happens during each cycle to make sure all required
         operations (primary IP assignment) happen during the same batch.
         """
-        if not self.settings.get("ipaddress"):
+        if not self.settings.get("ipam.ipaddress"):
             self.logger.log_info(
                 "Did not ask to sync ipaddresses, skipping.", obj=self.sync
             )
@@ -1102,40 +1035,45 @@ class IPFabricSyncRunner(object):
         records = self.preprocess_data(data=data)
 
         self.logger.log_info("Starting data sync.", obj=self.sync)
-        self.sync_items(
-            items=records["site"],
-            cf=self.sync.update_custom_fields,
-            ingestion=ingestion,
-        )
-        self.sync_items(items=records["manufacturer"])
-        self.sync_items(items=records["devicetype"])
-        self.sync_items(items=records["platform"])
-        self.sync_items(items=records["devicerole"])
+        try:
+            # This signal does not call for snapshot(), causing issue with branching plugin
+            signals.post_save.disconnect(sync_cached_scope_fields, sender=Site)
+            self.sync_items(
+                items=records["dcim.site"],
+                cf=self.sync.update_custom_fields,
+                ingestion=ingestion,
+            )
+        finally:
+            signals.post_save.connect(sync_cached_scope_fields, sender=Site)
+        self.sync_items(items=records["dcim.manufacturer"])
+        self.sync_items(items=records["dcim.devicetype"])
+        self.sync_items(items=records["dcim.platform"])
+        self.sync_items(items=records["dcim.devicerole"])
         try:
             # This signal does not call for snapshot(), causing issue with branching plugin
             signals.post_save.disconnect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
-            self.sync_items(items=records["virtualchassis"])
+            self.sync_items(items=records["dcim.virtualchassis"])
             self.sync_devices(
-                devices=records["device"],
+                devices=records["dcim.device"],
                 cf=self.sync.update_custom_fields,
                 ingestion=ingestion,
             )
             # The Device exists now, so we can update the master of the VC.
             # The logic is handled in transform maps.
-            self.sync_items(items=records["virtualchassis"], stats=False)
+            self.sync_items(items=records["dcim.virtualchassis"], stats=False)
         finally:
             signals.post_save.connect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
-        self.sync_items(items=records["interface"])
-        self.sync_items(items=records["macaddress"])
-        self.sync_items(items=records["inventoryitem"])
-        self.sync_items(items=records["vlan"])
-        self.sync_items(items=records["vrf"])
-        self.sync_items(items=records["prefix"])
-        self.sync_ip_addresses(ip_addresses=records["ipaddress"])
+        self.sync_items(items=records["dcim.interface"])
+        self.sync_items(items=records["dcim.macaddress"])
+        self.sync_items(items=records["dcim.inventoryitem"])
+        self.sync_items(items=records["ipam.vlan"])
+        self.sync_items(items=records["ipam.vrf"])
+        self.sync_items(items=records["ipam.prefix"])
+        self.sync_ip_addresses(ip_addresses=records["ipam.ipaddress"])
 
         # Make sure to clean queue (and memory) at the end
         self.events_clearer.clear()