ipfabric_netbox 4.3.2b9__py3-none-any.whl → 4.3.2b11__py3-none-any.whl

ipfabric_netbox/utilities/ipfutils.py

@@ -2,11 +2,8 @@ import json
 import logging
 from collections import Counter
 from copy import deepcopy
-from enum import Enum
 from functools import cache
-from functools import partial
 from importlib import metadata
-from typing import Any
 from typing import Callable
 from typing import TYPE_CHECKING
 from typing import TypeVar
@@ -14,8 +11,15 @@ from typing import TypeVar
 from core.exceptions import SyncError
 from core.signals import clear_events
 from dcim.models import Device
+from dcim.models import Site
 from dcim.models import VirtualChassis
 from dcim.signals import assign_virtualchassis_master
+
+try:
+    # Got added in NetBox 4.4.9
+    from dcim.signals import sync_cached_scope_fields
+except ImportError:
+    sync_cached_scope_fields = None
 from django.conf import settings
 from django.core.exceptions import MultipleObjectsReturned
 from django.core.exceptions import ObjectDoesNotExist
@@ -136,26 +140,6 @@ class IPFabric(object):
             "user-agent"
         ] += f'; ipfabric-netbox/{metadata.version("ipfabric-netbox")}'  # noqa: E702
 
-    def get_snapshots(self) -> dict:
-        formatted_snapshots = {}
-        if self.ipf:
-            for snapshot_ref, snapshot in self.ipf.snapshots.items():
-                if snapshot.status != "done" and snapshot.finish_status != "done":
-                    continue
-                if snapshot_ref in ["$prev", "$lastLocked"]:
-                    continue
-                if snapshot.name:
-                    description = (
-                        snapshot.name
-                        + " - "
-                        + snapshot.end.strftime("%d-%b-%y %H:%M:%S")
-                    )
-                else:
-                    description = snapshot.end.strftime("%d-%b-%y %H:%M:%S")
-
-                formatted_snapshots[snapshot_ref] = (description, snapshot.snapshot_id)
-        return formatted_snapshots
-
     def get_table_data(self, table, device):
         filter = {"sn": ["eq", device.serial]}
         split = table.split(".")
@@ -194,15 +178,13 @@ class DataRecord:
 
     def __init__(
         self,
-        app: str,
-        model: str,
+        model_string: str,
         data: dict,
         # These values are filled later as the record is passed down the pipeline
         context: dict | None = None,
         transform_map: "IPFabricTransformMap | None" = None,
     ):
-        self.app = app
-        self.model = model
+        self.model_string = model_string
         self.data = data
         self.context = context or dict()
         self.transform_map = transform_map
@@ -212,8 +194,7 @@ class DataRecord:
             try:
                 self._hash = hash(
                     (
-                        self.app,
-                        self.model,
+                        self.model_string,
                         # Since the dicts are already ordered, it is safe to hash them
                         # .values() are mutable, this is fixed by tuple() to get same hash every time
                         make_hashable(self.data),
@@ -248,119 +229,11 @@ class DataRecord:
         self._hash = None  # Invalidate cached hash
 
 
-# TODO: 1) Store this as model in DB linked to Transform map
-# TODO: 2) Each table will be added to template data with customizable name (instead of just `object`)
-# TODO: and linked using common column (like `sn` for Device and Interface)
-# TODO: 3) Only pull required tables according to sync settings
-class Endpoints(Enum):
-    SITE = "inventory.sites"
-    DEVICE = "inventory.devices"
-    VIRTUALCHASSIS = "technology.platforms.stacks_members"
-    INTERFACE = "inventory.interfaces"
-    INVENTORYITEM = "inventory.pn"
-    VLAN = "technology.vlans.site_summary"
-    VRF = "technology.routing.vrf_detail"
-    PREFIX = "technology.managed_networks.networks"
-    IPADDRESS = "technology.addressing.managed_ip_ipv4"
-
-
-class Filters(Enum):
-    SITE = "site"
-    DEVICE = "device"
-    VIRTUALCHASSIS = "virtualchassis"
-    INTERFACE = "interface"
-    INVENTORYITEM = "inventoryitem"
-    VLAN = "vlan"
-    VRF = "vrf"
-    PREFIX = "prefix"
-    IPADDRESS = "ipaddress"
-
-    @staticmethod
-    def _site_filter(sites: dict | None) -> dict:
-        if sites:
-            return {"or": [{"siteName": ["eq", site]} for site in sites]}
-        return {}
-
-    @staticmethod
-    def _device_filter(sites: dict | None, child_table: bool = False) -> dict:
-        key = "vendor" if not child_table else "device.vendor"
-        excluded_vendors = ["aws", "azure"]
-        device_filter = {"and": [{key: ["neq", vendor]} for vendor in excluded_vendors]}
-        if sites:
-            site_filter = Filters._site_filter(sites)
-            device_filter["and"].append(site_filter)
-        return device_filter
-
-    @staticmethod
-    def _virtualchassis_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _interface_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _inventoryitem_filter(sites: dict | None) -> dict:
-        inventory_item_filter = {
-            "and": [
-                {"sn": ["empty", False]},
-                {"name": ["empty", False]},
-            ]
-        }
-        if site_filter := Filters._device_filter(sites, child_table=True):
-            inventory_item_filter["and"].append(site_filter)
-        return inventory_item_filter
-
-    @staticmethod
-    def _vlan_filter(sites: dict | None) -> dict:
-        # Remove VLANs with ID 0, minimum VLAN ID in NetBox is 1
-        vlan_filter = {"and": [{"vlanId": ["neq", 0]}]}
-        if site_filter := Filters._site_filter(sites):
-            vlan_filter["and"].append(site_filter)
-        return vlan_filter
-
-    @staticmethod
-    def _vrf_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def _prefix_filter(sites: dict | None) -> dict:
-        if site_filter := Filters._device_filter(sites, child_table=True):
-            return {"and": [site_filter, {"and": [{"net": ["empty", False]}]}]}
-        else:
-            return {"and": [{"net": ["empty", False]}]}
-
-    @staticmethod
-    def _ipaddress_filter(sites: dict | None) -> dict:
-        return Filters._device_filter(sites, child_table=True)
-
-    @staticmethod
-    def get_filter(endpoint: str, sites: dict | None) -> dict:
-        method_name = f"_{getattr(Filters, endpoint).value}_filter"
-        filter_func = getattr(Filters, method_name, None)
-        if filter_func:
-            return filter_func(sites)
-        return {}
-
-
-# TODO: Store also hierarchy of models (e.g. Device required Device and virtual chassis endpoints)
-
-
-class EndpointHandler:
-    def __init__(self, client: IPFClient, snapshot_id: str) -> None:
-        self.client = client
-        self.snapshot_id = snapshot_id
-
-    def resolve_endpoint(self, endpoint: str) -> Callable[..., Any]:
-        """Resolve a dot-separated endpoint string to the corresponding IPFClient attribute."""
-        obj = self.client
-        for attr in endpoint.split("."):
-            obj = getattr(obj, attr)
-        return partial(obj.all, snapshot_id=self.snapshot_id)
-
-
-def order_members(members: list[dict]) -> dict[str, list[dict]]:
-    """Order VC members to dict, where key is master serial number and values are all members."""
+def order_members(members: list[dict], sn_column: str) -> dict[str, list[dict]]:
+    """Order stack members to dict, where key is master serial number and values are all members.
+    For stack members table we get `memberSn` column for member serial number.
+    For VSS chassis table we get `chassisSn` column for member serial number.
+    """
     devices = {}
 
     for member in members:
@@ -368,7 +241,7 @@ def order_members(members: list[dict]) -> dict[str, list[dict]]:
         # This can be spotted by checking if `sn` is different from `memberSn` for the master device
         # Plus `sn` will be IP of loopback in hex...
         master_serial = member.get("sn")
-        if master_serial and member.get("memberSn"):
+        if master_serial and member.get(sn_column):
             if master_serial in devices:
                 devices[master_serial].append(member)
             else:
@@ -400,17 +273,20 @@ def prepare_devices(
         if child_members := members.get(device.get("sn")):
             # This device is the VC master, and we're iterating over all it's members
             for child_member in child_members:
+                member_id = child_member.get("member") or child_member.get("chassisId")
+                member_sn = child_member.get("memberSn") or child_member.get(
+                    "chassisSn"
+                )
                 # There is physically no device with hostname matching the virtual chassis
                 # There are only members, so "hostname/1", "hostname/2", etc.
                 new_device = deepcopy(device)
-                new_device[
-                    "hostname"
-                ] = f"{device['hostname']}/{child_member.get('member')}"
+                new_device["hostname"] = f"{device['hostname']}/{member_id}"
                 new_device["virtual_chassis"] = child_member
-                if device.get("sn") != child_member.get("memberSn"):
+                if device.get("sn") != member_sn:
                     # VC members (non-master) are not in Device table, need to add them as new Device
-                    new_device["model"] = child_member.get("pn")
-                    new_device["sn"] = child_member.get("memberSn")
+                    # VSS Chassis have no model in the table, get it from master device
+                    new_device["model"] = child_member.get("pn") or device["model"]
+                    new_device["sn"] = member_sn
                     all_devices.append(new_device)
                 else:
                     # Master device, create the virtual chassis based on it
@@ -465,7 +341,7 @@ class IPFabricSyncRunner(object):
         exception: Exception,
         ingestion: "IPFabricIngestion",
         message: str = None,
-        model: str = None,
+        model_string: str = None,
         context: dict = None,
         data: dict = None,
     ) -> "tuple[bool, IPFabricIngestionIssue]":
@@ -476,7 +352,7 @@ class IPFabricSyncRunner(object):
 
         error_serial = self.get_error_serial(context, data)
         # Ignore models that do not have any dependencies by serial number
-        if error_serial and model not in ["ipaddress", "macaddress"]:
+        if error_serial and model_string not in ["ipam.ipaddress", "dcim.macaddress"]:
             self.error_serials.add(error_serial)
 
         # TODO: This is to prevent circular import issues, clean it up later.
@@ -487,7 +363,7 @@ class IPFabricSyncRunner(object):
                 ingestion=ingestion,
                 exception=exception.__class__.__name__,
                 message=message or getattr(exception, "message", str(exception)),
-                model=model,
+                model=model_string,
                 coalesce_fields={
                     k: v for k, v in context.items() if k not in ["defaults"]
                 },
@@ -515,19 +391,19 @@ class IPFabricSyncRunner(object):
                 # Logging section for logs inside job - facing user
                 self = args[0]
                 if isinstance(err, SearchError):
-                    if self.settings.get(err.model):
+                    if self.settings.get(err.model_string):
                         self.logger.log_failure(
-                            f"Aborting syncing `{err.model}` instance due to above error, please check your transform maps and/or existing data.",
+                            f"Aborting syncing `{err.model_string}` instance due to above error, please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                     else:
                         self.logger.log_failure(
-                            f"Syncing `{err.model}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
+                            f"Syncing `{err.model_string}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                 if isinstance(err, IPAddressDuplicateError):
                     self.logger.log_warning(
-                        f"IP Address `{err.data.get('address')}` already exists in `{err.model}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
+                        f"IP Address `{err.data.get('address')}` already exists in `{err.model_string}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
                         obj=self.sync,
                     )
                 else:
@@ -549,13 +425,11 @@ class IPFabricSyncRunner(object):
 
     def get_transform_context(self, record: DataRecord) -> DataRecord:
         if not record.transform_map:
-            raise SystemError(
-                f"No transform map available for {record.app}: {record.model}"
-            )
+            raise SystemError(f"No transform map available for {record.model_string}")
         try:
             record.context = record.transform_map.get_context(record.data)
         except Exception as err:
-            message = f"Error getting context for `{record.model}`."
+            message = f"Error getting context for `{record.model_string}`."
             if isinstance(err, ObjectDoesNotExist):
                 message += (
                     " Could not find related object using template in transform maps."
@@ -566,11 +440,14 @@ class IPFabricSyncRunner(object):
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 data=record.data,
             )
             raise SearchError(
-                message=message, data=record.data, model=record.model, issue_id=issue.pk
+                message=message,
+                data=record.data,
+                model_string=record.model_string,
+                issue_id=issue.pk,
             ) from err
 
         return record
@@ -581,67 +458,67 @@ class IPFabricSyncRunner(object):
         if error_serial and error_serial in self.error_serials:
             # We want to raise it as exception so it's shown in ingestion issues but can be filtered out.
             exception = RequiredDependencyFailedSkip(
-                message=f"Skipping syncing of `{record.model}` with serial `{error_serial}` due to previous errors.",
-                model=record.model,
+                message=f"Skipping syncing of `{record.model_string}` with serial `{error_serial}` due to previous errors.",
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             _, issue = self.create_or_get_sync_issue(
                 exception=exception,
                 ingestion=self.ingestion,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise exception
         record = self.get_transform_context(record)
         queryset = record.transform_map.target_model.model_class().objects
-        model_settings = self.settings.get(record.model, False)
+        model_settings = self.settings.get(f"{record.model_string}", False)
 
         obj = None
         try:
             connection_name = self.get_db_connection_name()
             if model_settings:
-                logger.info(f"Creating {record.model}")
+                logger.info(f"Creating {record.model_string}")
                 obj = record.transform_map.update_or_create_instance(
                     context=record.context,
                     tags=self.sync.tags.all(),
                     connection_name=connection_name,
                 )
             else:
-                logger.info(f"Getting {record.model}")
+                logger.info(f"Getting {record.model_string}")
                 record.context.pop("defaults", None)
                 obj = queryset.using(connection_name).get(**record.context)
         except queryset.model.DoesNotExist as err:
-            message = f"Instance of `{record.model}` not found."
+            message = f"Instance of `{record.model_string}` not found."
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
             ) from err
         except queryset.model.MultipleObjectsReturned as err:
-            message = f"Multiple instances of `{record.model}` found."
+            message = f"Multiple instances of `{record.model_string}` found."
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
@@ -650,12 +527,12 @@ class IPFabricSyncRunner(object):
             _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
             )
             raise SyncDataError(
-                model=record.model,
+                model_string=record.model_string,
                 context=record.context,
                 data=record.data,
                 issue_id=issue.pk,
@@ -664,12 +541,16 @@ class IPFabricSyncRunner(object):
         return obj
 
     def collect_data(self):
+        # Importing here to avoid circular import
+        from ..models import IPFabricEndpoint
+
         try:
             self.logger.log_info(
                 "Collecting information from IP Fabric",
                 obj=self.sync.snapshot_data.source,
             )
             data = {}
+            # TODO: Replace endpoint value in SnapshotData with Endpoint link
             if self.sync.snapshot_data.source.type == IPFabricSourceTypeChoices.REMOTE:
                 # This requires data already pushed to NetBox by user, does not connect to IPF directly
                 self.logger.log_info(
@@ -679,34 +560,68 @@ class IPFabricSyncRunner(object):
                     raise SyncError(
                         "No snapshot data available. This is a remote sync. Push data to NetBox first."
                     )
-                for endpoint in Endpoints:
-                    data[endpoint.name.lower()] = list(
+                for endpoint in IPFabricEndpoint.objects.all():
+                    data[endpoint.endpoint] = list(
                         self.sync.snapshot_data.ipf_data.filter(
-                            type=endpoint.name.lower()
+                            type=endpoint.endpoint
                         ).values_list("data", flat=True)
                     )
             else:
                 # This pulls data directly from IP Fabric instance
+                # TODO: If certain columns are not enabled in IPF, sync will fail in odd ways
+                #  For example missing `nameOriginal` on Interface makes it unable to sync IPs
+                #  When pulling from endpoint, make sure to specify columns according to transform maps
+                #  This can be pulled using logic in IPFabricTransformMap.strip_source_data()
+                # TODO: Also cache it to improve performance, no need to regex every time!
                 self.logger.log_info(
                     "Local collector being used for snapshot data.", obj=self.sync
                 )
-                endpoint_handler = EndpointHandler(
-                    self.client,
-                    snapshot_id=self.settings["snapshot_id"],
-                )
-                ingestion_sites = self.settings.get("sites")
-                for endpoint in Endpoints:
-                    filters = Filters.get_filter(endpoint.name, ingestion_sites)
+                enabled_models = [
+                    param
+                    for param in self.sync.parameters.keys()
+                    if "." in param and self.sync.parameters.get(param) is True
+                ]
+                # Always pull devices for now since other models depend on them
+                enabled_models.append("dcim.device")
+                # Interfaces are required for IP Addresses assigned interface name
+                if "ipam.ipaddress" in enabled_models:
+                    enabled_models.append("dcim.interface")
+                transform_maps = self.sync.get_transform_maps()
+                for endpoint in IPFabricEndpoint.objects.all():
+                    # FIXME: Dirty hack to sync VSS, remove when IN-68 is getting done
+                    if endpoint.endpoint == "/technology/platforms/vss/chassis":
+                        tms_for_endpoint = transform_maps.filter(
+                            source_endpoint__endpoint="/technology/platforms/stack/members"
+                        )
+                    else:
+                        tms_for_endpoint = transform_maps.filter(
+                            source_endpoint=endpoint
+                        )
+
+                    if not tms_for_endpoint.exists():
+                        raise SyncError(
+                            f"No transform map found for endpoint `{endpoint.endpoint}`."
+                        )
+                    if not [
+                        True
+                        for tm in tms_for_endpoint
+                        if ".".join(tm.target_model.natural_key()) in enabled_models
+                    ]:
+                        logger.debug(
+                            f"Skipping endpoint `{endpoint.endpoint}` as no enabled models are using it."
+                        )
+                        continue
+                    filters = endpoint.combine_filters(sync=self.sync)
                     logger.debug(
-                        f"Collecting data from endpoint: `{endpoint.value}` using filter `{json.dumps(filters)}`."
+                        f"Collecting data from endpoint: `{endpoint.endpoint}` using filter `{json.dumps(filters)}`."
                     )
-                    data[endpoint.name.lower()] = endpoint_handler.resolve_endpoint(
-                        endpoint.value
-                    )(
+                    data[endpoint.endpoint] = self.client.fetch_all(
+                        url=endpoint.endpoint,
+                        snapshot_id=self.settings["snapshot_id"],
                         filters=filters,
                     )
                     self.logger.log_info(
-                        f"Collected {len(data[endpoint.name.lower()])} items from endpoint `{endpoint.value}`.",
+                        f"Collected {len(data[endpoint.endpoint])} items from endpoint `{endpoint.endpoint}`.",
                         obj=self.sync.snapshot_data.source,
                     )
         except Exception as e:
@@ -717,103 +632,182 @@ class IPFabricSyncRunner(object):
         return data
 
     @cache
-    def get_transform_map(self, app: str, model: str) -> "IPFabricTransformMap":
+    def get_transform_map(
+        self, app: str, model: str, endpoint: str = None
+    ) -> "IPFabricTransformMap":
         """Get transform map for given app and model. Cached to improve performance."""
-        return self.transform_maps.get(
-            target_model__app_label=app, target_model__model=model
+        filter_kwargs = {
+            "target_model__app_label": app,
+            "target_model__model": model,
+        }
+        if endpoint:
+            filter_kwargs["source_endpoint__endpoint"] = endpoint
+        return self.transform_maps.get(**filter_kwargs)
+
+    def create_new_data_records(
+        self,
+        app: str,
+        model: str,
+        data: list[dict] | dict[str, list[dict]],
+        endpoint: str = None,
+    ) -> set[DataRecord]:
+        """Create DataRecord objects for given app, model and data list if enabled in settings."""
+        if not self.sync.parameters.get(f"{app}.{model}"):
+            return set()
+        if isinstance(data, dict):
+            # Data is either already list or full data where we have to choose the list
+            transform_map = self.get_transform_map(
+                app=app, model=model, endpoint=endpoint
+            )
+            data = data.get(transform_map.source_endpoint.endpoint, [])
+        return set(
+            self.create_new_data_record(
+                app=app, model=model, data=item, endpoint=endpoint
+            )
+            for item in data
         )
 
-    def create_new_data_record(self, app: str, model: str, data: dict) -> DataRecord:
+    def create_new_data_record(
+        self, app: str, model: str, data: dict, endpoint: str = None
+    ) -> DataRecord:
         """Extract only relevant source data according to transform map configuration."""
-        transform_map = self.get_transform_map(app=app, model=model)
+        transform_map = self.get_transform_map(app=app, model=model, endpoint=endpoint)
+        model_string = f"{app}.{model}"
         try:
             source_data = transform_map.strip_source_data(data)
         except KeyError as err:
             raise SyncError(
-                f"Missing key column {err.args[0]} in source data when preparing data for {app}_{model}."
+                f"Missing key column {err.args[0]} in source data when preparing data for {model_string}."
             ) from err
         return DataRecord(
-            app=app, model=model, data=source_data, transform_map=transform_map
+            model_string=model_string, data=source_data, transform_map=transform_map
         )
 
-    def preprocess_data(self, data: dict) -> dict[str, set[DataRecord]]:
-        # TODO: Only process data according to settings to improve performance
-        # Set those records that can't be iterated separately
-        # Others are as empty set to define order which is shown in UI progress
-        records = {
-            "site": set(
-                self.create_new_data_record(app="dcim", model="site", data=item)
-                for item in data.get("site", [])
-            ),
-            "manufacturer": set(),
-            "devicetype": set(),
-            "platform": set(),
-            "devicerole": set(),
-            "device": set(),
-            "virtualchassis": set(),
-            "interface": set(),
-            "macaddress": set(),
-            "inventoryitem": set(
-                self.create_new_data_record(
-                    app="dcim", model="inventoryitem", data=item
-                )
-                for item in data.get("inventoryitem", [])
-            ),
-            "vlan": set(
-                self.create_new_data_record(app="ipam", model="vlan", data=item)
-                for item in data.get("vlan", [])
-            ),
-            "vrf": set(
-                self.create_new_data_record(app="ipam", model="vrf", data=item)
-                for item in data.get("vrf", [])
-            ),
-            "prefix": set(
-                self.create_new_data_record(app="ipam", model="prefix", data=item)
-                for item in data.get("prefix", [])
-            ),
-            "ipaddress": set(),
+    def _get_custom_preprocessor(
+        self, model_string: str, endpoint: str
+    ) -> Callable | None:
+        """
+        Get custom preprocessing function for a model+endpoint combination if one exists.
+
+        Args:
+            model_string: The model in 'app.model' format (e.g., 'dcim.virtualchassis')
+            endpoint: The source endpoint (e.g., '/technology/platforms/stack/members')
+
+        Returns:
+            Callable preprocessor function or None if no custom preprocessor exists
+        """
+        preprocessors = {
+            # Devices and related models processed together (from /inventory/devices)
+            ("dcim.device", "/inventory/devices"): self._preprocess_devices,
+            ("dcim.manufacturer", "/inventory/devices"): self._preprocess_devices,
+            ("dcim.devicetype", "/inventory/devices"): self._preprocess_devices,
+            ("dcim.platform", "/inventory/devices"): self._preprocess_devices,
+            ("dcim.devicerole", "/inventory/devices"): self._preprocess_devices,
+            # Virtual chassis from stack members
+            (
+                "dcim.virtualchassis",
+                "/technology/platforms/stack/members",
+            ): self._preprocess_devices,
+            # Interfaces and MAC addresses processed together
+            ("dcim.interface", "/inventory/interfaces"): self._preprocess_interfaces,
+            ("dcim.macaddress", "/inventory/interfaces"): self._preprocess_interfaces,
+            # IP addresses need interface data
+            (
+                "ipam.ipaddress",
+                "/technology/addressing/managed-ip/ipv4",
+            ): self._preprocess_ipaddresses,
         }
 
+        return preprocessors.get((model_string, endpoint))
+
+    def _preprocess_devices(self, data: dict, records: dict) -> None:
+        """Custom preprocessing for devices - single iteration for efficiency."""
+        if not (
+            self.sync.parameters.get("dcim.device")
+            or self.sync.parameters.get("dcim.virtualchassis")
+            or self.sync.parameters.get("dcim.interface")
+            or self.sync.parameters.get("dcim.macaddress")
+            or self.sync.parameters.get("ipam.ipaddress")
+        ):
+            return
+
+        # Prepare virtual chassis members once
+        members = {}
+        if self.sync.parameters.get("dcim.virtualchassis"):
+            self.logger.log_info("Preparing virtual chassis members", obj=self.sync)
+            members = order_members(
+                data.get("/technology/platforms/stack/members", []), "memberSn"
+            )
+            members.update(
+                order_members(
+                    data.get("/technology/platforms/vss/chassis", []), "chassisSn"
+                )
+            )
+
         self.logger.log_info("Preparing devices", obj=self.sync)
-        members = order_members(data.get("virtualchassis", []))
-        devices, virtualchassis = prepare_devices(data.get("device", []), members)
+        devices, virtualchassis = prepare_devices(
+            data.get("/inventory/devices", []), members
+        )
 
-        # We need to store primary IPs of Devices to assign them later
-        # since they are not stored on Device object directly
-        # TODO: This will be later replaced when we are able to sync from multiple API tables to 1 model
         device_primary_ips = {}
 
+        # Single iteration creating records for all related models
         for device in devices:
-            records["manufacturer"].add(
-                self.create_new_data_record(
-                    app="dcim", model="manufacturer", data=device
+            if self.sync.parameters.get("dcim.manufacturer"):
+                records["dcim.manufacturer"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="manufacturer", data=device
+                    )
                 )
-            )
-            records["devicetype"].add(
-                self.create_new_data_record(app="dcim", model="devicetype", data=device)
-            )
-            records["platform"].add(
-                self.create_new_data_record(app="dcim", model="platform", data=device)
-            )
-            records["devicerole"].add(
-                self.create_new_data_record(app="dcim", model="devicerole", data=device)
-            )
-            # This field is required by Device transform maps, but is set only when Device is part of VC.
+
+            if self.sync.parameters.get("dcim.devicetype"):
+                records["dcim.devicetype"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="devicetype", data=device
+                    )
+                )
+
+            if self.sync.parameters.get("dcim.platform"):
+                records["dcim.platform"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="platform", data=device
+                    )
+                )
+
+            if self.sync.parameters.get("dcim.devicerole"):
+                records["dcim.devicerole"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="devicerole", data=device
+                    )
+                )
+
             if "virtual_chassis" not in device:
                 device["virtual_chassis"] = None
-            records["device"].add(
-                self.create_new_data_record(app="dcim", model="device", data=device)
-            )
+
+            if self.sync.parameters.get("dcim.device"):
+                records["dcim.device"].add(
+                    self.create_new_data_record(app="dcim", model="device", data=device)
+                )
+
             device_primary_ips[device.get("sn")] = device.get("loginIp")
 
-        records["virtualchassis"] = set(
-            self.create_new_data_record(app="dcim", model="virtualchassis", data=item)
-            for item in virtualchassis
+        self._device_primary_ips = device_primary_ips
+        records["dcim.virtualchassis"] = self.create_new_data_records(
+            app="dcim", model="virtualchassis", data=virtualchassis
         )
 
-        # `nameOriginal` is human-readable interface name hidden column in IP Fabric
-        # This allows us to use it instead of the `intName`
-        # But it can be customized using transform maps, so we need to use the current value
+    def _preprocess_interfaces(self, data: dict, records: dict) -> None:
+        """Custom preprocessing for interfaces and MAC addresses."""
+        if not (
+            self.sync.parameters.get("dcim.interface")
+            or self.sync.parameters.get("dcim.macaddress")
+            or self.sync.parameters.get("ipam.ipaddress")
+        ):
+            return
+
+        if not hasattr(self, "_device_primary_ips"):
+            self._preprocess_devices(data=data, records=records)
+
         interface_key = "nameOriginal"
         try:
             int_transform_map = self.get_transform_map(app="dcim", model="interface")
@@ -821,58 +815,132 @@ class IPFabricSyncRunner(object):
             interface_key = int_name_field_map.source_field
         except Exception as e:
             self.logger.log_failure(
-                f"Error collecting information about transform map for interface name: {e}",
-                obj=self.sync,
+                f"Error collecting transform map info: {e}", obj=self.sync
             )
             raise SyncError(f"Error collecting source column name for interface: {e}")
 
-        self.logger.log_info("Preparing Interfaces", obj=self.sync)
         # Store human-readable interface names to use them later for IP Addresses
         readable_int_names = {}
-        for interface in data["interface"]:
-            interface_record = self.create_new_data_record(
-                app="dcim", model="interface", data=interface
-            )
-            interface_record.data["loginIp"] = device_primary_ips.get(
-                interface.get("sn")
-            )
-            records["interface"].add(interface_record)
+        self.logger.log_info("Preparing Interfaces", obj=self.sync)
+
+        for interface in data.get("/inventory/interfaces", []):
+            if self.sync.parameters.get("dcim.interface"):
+                interface_record = self.create_new_data_record(
+                    app="dcim", model="interface", data=interface
+                )
+                interface_record.data["loginIp"] = self._device_primary_ips.get(
+                    interface.get("sn")
+                )
+                records["dcim.interface"].add(interface_record)
+
             readable_int_names[
                 f"{interface.get('sn')}_{interface.get('intName')}"
             ] = interface.get(interface_key)
-            records["macaddress"].add(
-                self.create_new_data_record(
-                    app="dcim", model="macaddress", data=interface
+
+            if self.sync.parameters.get("dcim.macaddress"):
+                records["dcim.macaddress"].add(
+                    self.create_new_data_record(
+                        app="dcim", model="macaddress", data=interface
+                    )
                 )
-            )
+
+        self._readable_int_names = readable_int_names
+
+    def _preprocess_ipaddresses(self, data: dict, records: dict) -> None:
+        """Custom preprocessing for IP addresses."""
+        if not self.sync.parameters.get("ipam.ipaddress"):
+            return
+
+        if not hasattr(self, "_readable_int_names"):
+            self._preprocess_interfaces(data=data, records=records)
 
         self.logger.log_info("Preparing IP Addresses", obj=self.sync)
-        for ip in data["ipaddress"]:
-            # We get `nameOriginal` from Interface table to get human-readable name instead fo `intName`
-            ip["nameOriginal"] = readable_int_names.get(
+        for ip in data.get("/technology/addressing/managed-ip/ipv4", []):
+            ip["nameOriginal"] = self._readable_int_names.get(
                 f"{ip.get('sn')}_{ip.get('intName')}"
             )
-            # Let's skip IPs we cannot assign to an interface
             if not ip["nameOriginal"]:
                 continue
+
             ipaddress_record = self.create_new_data_record(
                 app="ipam", model="ipaddress", data=ip
             )
-            # Store whether this IP is primary for the device
             ipaddress_record.data["is_primary"] = ip.get(
                 "sn"
-            ) in device_primary_ips and device_primary_ips.get(
+            ) in self._device_primary_ips and self._device_primary_ips.get(
                 ip.get("sn")
             ) == ipaddress_record.data.get(
                 "ip"
             )
-            records["ipaddress"].add(ipaddress_record)
+            records["ipam.ipaddress"].add(ipaddress_record)
+
+    def preprocess_data(self, data: dict) -> dict[str, set[DataRecord]]:
+        """
+        Preprocess data using hierarchical transform map order.
+
+        Iterates over transform maps in dependency order (parents before children),
+        using custom preprocessing functions where available or direct record creation.
+        Each transform map is processed exactly once, avoiding duplicate iterations.
+        """
+        # Get transform maps in hierarchical order (now returns transform maps directly!)
+        group_ids = self.sync.parameters.get("groups", [])
+        transform_maps_hierarchy = self.sync.get_model_hierarchy(group_ids=group_ids)
+
+        # Initialize records dict for all models (extract unique model strings)
+        records = {}
+        seen_models = set()
+        for transform_map in transform_maps_hierarchy:
+            model_string = f"{transform_map.target_model.app_label}.{transform_map.target_model.model}"
+            if model_string not in seen_models:
+                records[model_string] = set()
+                seen_models.add(model_string)
+
+        # Track which custom preprocessors have run (to avoid duplication)
+        # Use (preprocessor, endpoint) tuple to allow same preprocessor for different endpoints
+        preprocessors_run = set()
+
+        # Process each transform map in hierarchical order
+        for transform_map in transform_maps_hierarchy:
+            app = transform_map.target_model.app_label
+            model = transform_map.target_model.model
+            model_string = f"{app}.{model}"
+            endpoint = transform_map.source_endpoint.endpoint
+
+            # Skip if not enabled in sync parameters
+            if not self.sync.parameters.get(model_string):
+                continue
+
+            # Check for custom preprocessor (pass endpoint for endpoint-specific preprocessing)
+            preprocessor = self._get_custom_preprocessor(model_string, endpoint)
+
+            # Create unique key combining preprocessor function and endpoint
+            preprocessor_key = (preprocessor, endpoint) if preprocessor else None
+
+            if preprocessor and preprocessor_key not in preprocessors_run:
+                # Run custom preprocessor (may populate multiple models)
+                self.logger.log_info(
+                    f"Running custom preprocessor for `{model_string}` from `{endpoint}`",
+                    obj=self.sync,
+                )
+                preprocessor(data, records)
+                preprocessors_run.add(preprocessor_key)
+            elif not preprocessor:
+                # Standard processing: use the transform map directly (no need to filter!)
+                self.logger.log_info(
+                    f"Preparing `{model_string}` from `{endpoint}`", obj=self.sync
+                )
+                records[model_string].update(
+                    self.create_new_data_records(
+                        app=app, model=model, data=data, endpoint=endpoint
+                    )
+                )
 
-        for model, records_set in records.items():
-            if self.settings.get(model) and len(records_set):
-                self.logger.init_statistics(model, len(records_set))
+        # Log statistics
+        for model_string, records_set in records.items():
+            if self.settings.get(model_string) and len(records_set):
+                self.logger.init_statistics(model_string, len(records_set))
                 self.logger.log_info(
-                    f"Prepared {len(records_set)} items for `{model}` to be synced.",
+                    f"Prepared {len(records_set)} items for `{model_string}` to be synced.",
                     obj=self.sync,
                 )
 
@@ -883,13 +951,8 @@ class IPFabricSyncRunner(object):
         self,
         record: DataRecord,
         stats: bool = True,
-        sync: bool = False,
     ) -> ModelTypeVar | None:
         """Sync a single item to NetBox."""
-        # The `sync` param is a workaround since we need to get some models (Device...) even when not syncing them.
-        if not sync:
-            return None
-
         if not record.data:
             return None
 
@@ -897,7 +960,7 @@ class IPFabricSyncRunner(object):
 
         # Only log when we successfully synced the item and asked for it
         if stats and instance:
-            self.logger.increment_statistics(model=record.model)
+            self.logger.increment_statistics(model_string=record.model_string)
 
         return instance
 
@@ -909,11 +972,7 @@ class IPFabricSyncRunner(object):
         stats: bool = True,
     ) -> ModelTypeVar | None:
         """Sync a single item to NetBox."""
-        synced_object = self.sync_model(
-            record=record,
-            sync=self.settings.get(record.model),
-            stats=stats,
-        )
+        synced_object = self.sync_model(record=record, stats=stats)
         if synced_object is None:
             return None
 
@@ -939,10 +998,10 @@ class IPFabricSyncRunner(object):
         if not items:
             return
 
-        app, model = (lambda x: (x.app, x.model))(next(iter(items)))
-        if not self.settings.get(model):
+        model_string = (lambda record: record.model_string)(next(iter(items)))
+        if not self.settings.get(model_string):
             self.logger.log_info(
-                f"Did not ask to sync {model}s, skipping.", obj=self.sync
+                f"Did not ask to sync {model_string}s, skipping.", obj=self.sync
             )
             return
 
@@ -950,56 +1009,6 @@ class IPFabricSyncRunner(object):
             self.sync_item(item, cf, ingestion, stats)
             self.events_clearer.increment()
 
-    @handle_errors
-    def sync_devices(
-        self,
-        devices: set[DataRecord],
-        cf: bool = False,
-        ingestion: "IPFabricIngestion" = None,
-    ) -> None:
-        """Sync devices separately to handle resetting primary IP."""
-        if not self.settings.get("device"):
-            self.logger.log_info(
-                "Did not ask to sync devices, skipping.", obj=self.sync
-            )
-            return
-
-        for device in devices:
-            device_obj: "Device | None" = self.sync_item(
-                record=device, cf=cf, ingestion=ingestion
-            )
-
-            if (
-                device_obj is None
-                or device_obj.primary_ip4 is None
-                or device.data.get("loginIp") is not None
-            ):
-                self.events_clearer.increment()
-                continue
-
-            # If device has primary IP assigned in NetBox, but not in IP Fabric, remove it
-            try:
-                connection_name = self.get_db_connection_name()
-                device_obj.refresh_from_db(using=connection_name)
-                device_obj.snapshot()
-                device_obj.primary_ip4 = None
-                device_obj.save(using=connection_name)
-            except Exception as err:
-                _, issue = self.create_or_get_sync_issue(
-                    exception=err,
-                    ingestion=self.ingestion,
-                    message="Error removing primary IP current device.",
-                    model=device.model,
-                    data=device.data,
-                )
-                self.events_clearer.increment()
-                raise IPAddressPrimaryRemovalError(
-                    data=device.data,
-                    model=device.model,
-                    issue_id=issue.pk,
-                ) from err
-            self.events_clearer.increment()
-
     @handle_errors
     def sync_ipaddress(self, ip_address: DataRecord) -> "IPAddress | None":
         """Sync a single IP Address to NetBox, separated to use @handle_errors."""
@@ -1032,13 +1041,13 @@ class IPFabricSyncRunner(object):
                 exception=err,
                 ingestion=self.ingestion,
                 message="Error removing primary IP from other device.",
-                model=ip_address.model,
+                model_string=ip_address.model_string,
                 data=ip_address.data,
             )
             self.events_clearer.increment()
             raise IPAddressPrimaryRemovalError(
                 data=ip_address.data,
-                model=ip_address.model,
+                model_string=ip_address.model_string,
                 issue_id=issue.pk,
             ) from err
 
@@ -1063,13 +1072,13 @@ class IPFabricSyncRunner(object):
                     exception=err,
                     ingestion=self.ingestion,
                     message="Error assigning primary IP to device.",
-                    model=ip_address.model,
+                    model_string=ip_address.model_string,
                     data=ip_address.data,
                 )
                 self.events_clearer.increment()
                 raise IPAddressPrimaryAssignmentError(
                     data=ip_address.data,
-                    model=ip_address.model,
+                    model_string=ip_address.model_string,
                     issue_id=issue.pk,
                 ) from err
         self.events_clearer.increment()
@@ -1086,7 +1095,7 @@ class IPFabricSyncRunner(object):
         Cleaning events queue happens during each cycle to make sure all required
         operations (primary IP assignment) happen during the same batch.
         """
-        if not self.settings.get("ipaddress"):
+        if not self.settings.get("ipam.ipaddress"):
             self.logger.log_info(
                 "Did not ask to sync ipaddresses, skipping.", obj=self.sync
             )
@@ -1102,40 +1111,47 @@ class IPFabricSyncRunner(object):
         records = self.preprocess_data(data=data)
 
         self.logger.log_info("Starting data sync.", obj=self.sync)
-        self.sync_items(
-            items=records["site"],
-            cf=self.sync.update_custom_fields,
-            ingestion=ingestion,
-        )
-        self.sync_items(items=records["manufacturer"])
-        self.sync_items(items=records["devicetype"])
-        self.sync_items(items=records["platform"])
-        self.sync_items(items=records["devicerole"])
+        try:
+            # This signal does not call for snapshot(), causing issue with branching plugin
+            if sync_cached_scope_fields is not None:
+                signals.post_save.disconnect(sync_cached_scope_fields, sender=Site)
+            self.sync_items(
+                items=records["dcim.site"],
+                cf=self.sync.update_custom_fields,
+                ingestion=ingestion,
+            )
+        finally:
+            if sync_cached_scope_fields is not None:
+                signals.post_save.connect(sync_cached_scope_fields, sender=Site)
+        self.sync_items(items=records["dcim.manufacturer"])
+        self.sync_items(items=records["dcim.devicetype"])
+        self.sync_items(items=records["dcim.platform"])
+        self.sync_items(items=records["dcim.devicerole"])
         try:
             # This signal does not call for snapshot(), causing issue with branching plugin
             signals.post_save.disconnect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
-            self.sync_items(items=records["virtualchassis"])
-            self.sync_devices(
-                devices=records["device"],
+            self.sync_items(items=records["dcim.virtualchassis"])
+            self.sync_items(
+                items=records["dcim.device"],
                 cf=self.sync.update_custom_fields,
                 ingestion=ingestion,
             )
             # The Device exists now, so we can update the master of the VC.
             # The logic is handled in transform maps.
-            self.sync_items(items=records["virtualchassis"], stats=False)
+            self.sync_items(items=records["dcim.virtualchassis"], stats=False)
         finally:
             signals.post_save.connect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
-        self.sync_items(items=records["interface"])
-        self.sync_items(items=records["macaddress"])
-        self.sync_items(items=records["inventoryitem"])
-        self.sync_items(items=records["vlan"])
-        self.sync_items(items=records["vrf"])
-        self.sync_items(items=records["prefix"])
-        self.sync_ip_addresses(ip_addresses=records["ipaddress"])
+        self.sync_items(items=records["dcim.interface"])
+        self.sync_items(items=records["dcim.macaddress"])
+        self.sync_items(items=records["dcim.inventoryitem"])
+        self.sync_items(items=records["ipam.vlan"])
+        self.sync_items(items=records["ipam.vrf"])
+        self.sync_items(items=records["ipam.prefix"])
+        self.sync_ip_addresses(ip_addresses=records["ipam.ipaddress"])
 
         # Make sure to clean queue (and memory) at the end
         self.events_clearer.clear()