ipfabric_netbox 4.3.2b1__py3-none-any.whl → 4.3.2b2__py3-none-any.whl

ipfabric_netbox/utilities/ipfutils.py

@@ -1,14 +1,18 @@
 import json
 import logging
+from collections import Counter
+from copy import deepcopy
+from enum import Enum
+from functools import cache
+from functools import partial
 from importlib import metadata
+from typing import Any
 from typing import Callable
 from typing import TYPE_CHECKING
 from typing import TypeVar
 
 from core.exceptions import SyncError
 from dcim.models import Device
-from dcim.models import Interface
-from dcim.models import MACAddress
 from django.conf import settings
 from django.core.exceptions import MultipleObjectsReturned
 from django.core.exceptions import ObjectDoesNotExist
@@ -22,17 +26,17 @@ from netutils.utils import jinja2_convenience_function
 
 from ..choices import IPFabricSourceTypeChoices
 from ..choices import IPFabricSyncStatusChoices
-from ..exceptions import create_or_get_sync_issue
 from ..exceptions import IPAddressDuplicateError
+from ..exceptions import IPAddressPrimaryRemovalError
 from ..exceptions import SearchError
 from ..exceptions import SyncDataError
-from .nbutils import device_serial_max_length
-from .nbutils import order_devices
-from .nbutils import order_members
+
 
 if TYPE_CHECKING:
     from ..models import IPFabricIngestion
+    from ..models import IPFabricIngestionIssue
     from ipam.models import IPAddress
+    from ..models import IPFabricTransformMap
 
 logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
 
@@ -43,11 +47,14 @@ def slugify_text(value):
     return slugify(value)
 
 
-def serial(value):
-    sn_length = len(value.get("sn"))
-    serial_number = value.get("sn") if sn_length < device_serial_max_length else ""
+device_serial_max_length = Device._meta.get_field("serial").max_length
+
+
+def serial(data: dict) -> str:
+    sn_length = len(data.get("sn"))
+    serial_number = data.get("sn") if sn_length < device_serial_max_length else ""
     if not serial_number:
-        serial_number = value.get("id")
+        serial_number = data.get("id")
     return serial_number
 
 
@@ -97,16 +104,6 @@ class IPFabric(object):
                 formatted_snapshots[snapshot_ref] = (description, snapshot.snapshot_id)
         return formatted_snapshots
 
-    def get_sites(self, snapshot=None) -> list:
-        if snapshot:
-            raw_sites = self.ipf.inventory.sites.all(snapshot_id=snapshot)
-        else:
-            raw_sites = self.ipf.inventory.sites.all()
-        sites = []
-        for item in raw_sites:
-            sites.append(item["siteName"])
-        return sites
-
     def get_table_data(self, table, device):
         filter = {"sn": ["eq", device.serial]}
         split = table.split(".")
@@ -131,11 +128,246 @@ class IPFabric(object):
         return data, columns
 
 
+def make_hashable(obj):
+    if isinstance(obj, dict):
+        return tuple((k, make_hashable(v)) for k, v in obj.items())
+    elif isinstance(obj, list):
+        return tuple(make_hashable(x) for x in obj)
+    else:
+        return obj
+
+
+class DataRecord:
+    """Contains all data required to sync single object to NetBox."""
+
+    def __init__(
+        self,
+        app: str,
+        model: str,
+        data: dict,
+        # These values are filled later as the record is passed down the pipeline
+        context: dict | None = None,
+        transform_map: "IPFabricTransformMap | None" = None,
+    ):
+        self.app = app
+        self.model = model
+        self.data = data
+        self.context = context or dict()
+        self.transform_map = transform_map
+
+    def __hash__(self):
+        if self._hash is None:
+            try:
+                self._hash = hash(
+                    (
+                        self.app,
+                        self.model,
+                        # Since the dicts are already ordered, it is safe to hash them
+                        # .values() are mutable, this is fixed by tuple() to get same hash every time
+                        make_hashable(self.data),
+                        make_hashable(self.context),
+                    )
+                )
+            except Exception as err:
+                raise Exception(f"DATA: {self.data}") from err
+        return self._hash
+
+    def __eq__(self, other):
+        return isinstance(other, DataRecord) and hash(self) == hash(other)
+
+    # Make sure data and context are sorted by keys when stored to speed up hash calculation
+    # This should be safe since they do not contain nested dicts
+    @property
+    def data(self):
+        return self._data
+
+    @data.setter
+    def data(self, value):
+        self._data = {k: v for k, v in sorted(value.items())}
+        self._hash = None  # Invalidate cached hash
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = {k: v for k, v in sorted(value.items())}
+        self._hash = None  # Invalidate cached hash
+
+
+# TODO: 1) Store this as model in DB linked to Transform map
+# TODO: 2) Each table will be added to template data with customizable name (instead of just `object`)
+# TODO: and linked using common column (like `sn` for Device and Interface)
+# TODO: 3) Only pull required tables according to sync settings
+class Endpoints(Enum):
+    SITE = "inventory.sites"
+    DEVICE = "inventory.devices"
+    VIRTUALCHASSIS = "technology.platforms.stacks_members"
+    INTERFACE = "inventory.interfaces"
+    INVENTORYITEM = "inventory.pn"
+    VLAN = "technology.vlans.site_summary"
+    VRF = "technology.routing.vrf_detail"
+    PREFIX = "technology.managed_networks.networks"
+    IPADDRESS = "technology.addressing.managed_ip_ipv4"
+
+
+class Filters(Enum):
+    SITE = "site"
+    DEVICE = "device"
+    VIRTUALCHASSIS = "virtualchassis"
+    INTERFACE = "interface"
+    INVENTORYITEM = "inventoryitem"
+    VLAN = "vlan"
+    VRF = "vrf"
+    PREFIX = "prefix"
+    IPADDRESS = "ipaddress"
+
+    @staticmethod
+    def _site_filter(sites: dict | None) -> dict:
+        if sites:
+            return {"or": [{"siteName": ["eq", site]} for site in sites]}
+        return {}
+
+    @staticmethod
+    def _device_filter(sites: dict | None, child_table: bool = False) -> dict:
+        key = "vendor" if not child_table else "device.vendor"
+        excluded_vendors = ["aws", "azure"]
+        device_filter = {"and": [{key: ["neq", vendor]} for vendor in excluded_vendors]}
+        if sites:
+            site_filter = Filters._site_filter(sites)
+            device_filter["and"].append(site_filter)
+        return device_filter
+
+    @staticmethod
+    def _virtualchassis_filter(sites: dict | None) -> dict:
+        return Filters._device_filter(sites, child_table=True)
+
+    @staticmethod
+    def _interface_filter(sites: dict | None) -> dict:
+        return Filters._device_filter(sites, child_table=True)
+
+    @staticmethod
+    def _inventoryitem_filter(sites: dict | None) -> dict:
+        inventory_item_filter = {
+            "and": [
+                {"sn": ["empty", False]},
+                {"name": ["empty", False]},
+            ]
+        }
+        if site_filter := Filters._device_filter(sites, child_table=True):
+            inventory_item_filter["and"].append(site_filter)
+        return inventory_item_filter
+
+    @staticmethod
+    def _vlan_filter(sites: dict | None) -> dict:
+        # Remove VLANs with ID 0, minimum VLAN ID in NetBox is 1
+        vlan_filter = {"and": [{"vlanId": ["neq", 0]}]}
+        if site_filter := Filters._site_filter(sites):
+            vlan_filter["and"].append(site_filter)
+        return vlan_filter
+
+    @staticmethod
+    def _vrf_filter(sites: dict | None) -> dict:
+        return Filters._device_filter(sites, child_table=True)
+
+    @staticmethod
+    def _prefix_filter(sites: dict | None) -> dict:
+        if site_filter := Filters._device_filter(sites, child_table=True):
+            return {"and": [site_filter, {"and": [{"net": ["empty", False]}]}]}
+        else:
+            return {"and": [{"net": ["empty", False]}]}
+
+    @staticmethod
+    def _ipaddress_filter(sites: dict | None) -> dict:
+        return Filters._device_filter(sites, child_table=True)
+
+    @staticmethod
+    def get_filter(endpoint: str, sites: dict | None) -> dict:
+        method_name = f"_{getattr(Filters, endpoint).value}_filter"
+        filter_func = getattr(Filters, method_name, None)
+        if filter_func:
+            return filter_func(sites)
+        return {}
+
+
+# TODO: Store also hierarchy of models (e.g. Device required Device and virtual chassis endpoints)
+
+
+class EndpointHandler:
+    def __init__(self, client: IPFClient, snapshot_id: str) -> None:
+        self.client = client
+        self.snapshot_id = snapshot_id
+
+    def resolve_endpoint(self, endpoint: str) -> Callable[..., Any]:
+        """Resolve a dot-separated endpoint string to the corresponding IPFClient attribute."""
+        obj = self.client
+        for attr in endpoint.split("."):
+            obj = getattr(obj, attr)
+        return partial(obj.all, snapshot_id=self.snapshot_id)
+
+
+def order_members(members: list[dict]) -> dict[str, list[dict]]:
+    """Order VC members to dict, where key is master serial number and values are all members."""
+    devices = {}
+
+    for member in members:
+        # Caution: If the snapshot is created in development mode, the `sn` field is calculated from loopback IP
+        # This can be spotted by checking if `sn` is different from `memberSn` for the master device
+        # Plus `sn` will be IP of loopback in hex...
+        master_serial = member.get("sn")
+        if master_serial and member.get("memberSn"):
+            if master_serial in devices:
+                devices[master_serial].append(member)
+            else:
+                devices[master_serial] = [member]
+
+    return devices
+
+
+def order_devices(
+    devices: list[dict], members: dict[str, list[dict]]
+) -> tuple[list[dict], list[dict]]:
+    hostnames = [d["hostname"] for d in devices]
+    counter = Counter(hostnames)
+
+    new_devices = []
+    ordered_devices = []
+    virtualchassis = []
+
+    for device in devices[:]:
+        if counter[device["hostname"]] > 1:
+            device["hostname"] = f"{device['hostname']} - ({device['sn']})"
+        if child_members := members.get(device.get("sn")):
+            # This device is the VC master, and we're iterating over all it's members
+            for child_member in child_members:
+                if device.get("sn") != child_member.get("memberSn"):
+                    # VC members (non-master) are not in Device table, need to add them as new Device
+                    new_device = deepcopy(device)
+                    new_device[
+                        "hostname"
+                    ] = f"{device['hostname']}/{child_member.get('member')}"
+                    new_device["model"] = child_member.get("pn")
+                    new_device["sn"] = child_member.get("memberSn")
+                    new_device["virtual_chassis"] = child_member
+                    new_devices.append(new_device)
+                else:
+                    # This is the master device
+                    device["virtual_chassis"] = child_member
+                virtualchassis.append(child_member)
+            hostnames = [d["hostname"] for d in devices]
+            counter = Counter(hostnames)
+        else:
+            ordered_devices.append(device)
+    ordered_devices.extend(new_devices)
+    return ordered_devices, virtualchassis
+
+
 class IPFabricSyncRunner(object):
     def __init__(
         self,
         sync,
-        client: IPFabric = None,
+        client: IPFClient = None,
         ingestion=None,
         settings: dict = None,
     ) -> None:
@@ -150,6 +382,62 @@ class IPFabricSyncRunner(object):
         if self.sync.snapshot_data.status != "loaded":
             raise SyncError("Snapshot not loaded in IP Fabric.")
 
+        # Some objects depend on others being synced first, store errors to avoid duplicates
+        # We should store all dependant objects, but it's very hard to do
+        # For now store only serial numbers since that is the largest dependency chain
+        self.error_serials = set()
+
+    @staticmethod
+    def get_error_serial(context: dict | None, data: dict | None) -> str | None:
+        """Get error serial from context or raw data for skipping purposes."""
+        context = context or {}
+        data = data or {}
+        return (
+            context.get("sn") or context.get("serial") or serial(data)
+            if "sn" in data
+            else None
+        )
+
+    def create_or_get_sync_issue(
+        self,
+        exception: Exception,
+        ingestion: "IPFabricIngestion",
+        message: str = None,
+        model: str = None,
+        context: dict = None,
+        data: dict = None,
+    ) -> "tuple[bool, IPFabricIngestionIssue]":
+        """
+        Helper function to handle sync errors and create IPFabricIngestionIssue if needed.
+        """
+        context = context or {}
+
+        error_serial = self.get_error_serial(context, data)
+        if error_serial:
+            self.error_serials.add(error_serial)
+
+        # TODO: This is to prevent circular import issues, clean it up later.
+        from ..models import IPFabricIngestionIssue
+
+        if not hasattr(exception, "issue_id") or not exception.issue_id:
+            issue = IPFabricIngestionIssue.objects.create(
+                ingestion=ingestion,
+                exception=exception.__class__.__name__,
+                message=message or getattr(exception, "message", str(exception)),
+                model=model,
+                coalesce_fields={
+                    k: v for k, v in context.items() if k not in ["defaults"]
+                },
+                defaults=context.get("defaults", dict()),
+                raw_data=data or dict(),
+            )
+            if hasattr(exception, "issue_id"):
+                exception.issue_id = issue.id
+            return True, issue
+        else:
+            issue = IPFabricIngestionIssue.objects.get(id=exception.issue_id)
+            return False, issue
+
     @staticmethod
     def handle_errors(func: Callable):
         def wrapper(*args, **kwargs):
@@ -196,99 +484,111 @@ class IPFabricSyncRunner(object):
             connection_name = self.ingestion.branch.connection_name
         return connection_name
 
-    def get_model_or_update(self, app, model, data):
-        transform_map = self.transform_maps.get(
-            target_model__app_label=app, target_model__model=model
-        )
-
-        if not transform_map:
-            raise SystemError(f"No transform map available for {app}: {model}")
-
-        model_settings = self.settings.get(model, False)
+    def get_transform_context(self, record: DataRecord) -> DataRecord:
+        if not record.transform_map:
+            raise SystemError(
+                f"No transform map available for {record.app}: {record.model}"
+            )
         try:
-            context = transform_map.get_context(data)
+            record.context = record.transform_map.get_context(record.data)
         except Exception as err:
-            message = f"Error getting context for `{model}`."
+            message = f"Error getting context for `{record.model}`."
             if isinstance(err, ObjectDoesNotExist):
                 message += (
                     " Could not find related object using template in transform maps."
                 )
             elif isinstance(err, MultipleObjectsReturned):
                 message += " Multiple objects returned using on template in transform maps, the template is not strict enough."
-            _, issue = create_or_get_sync_issue(
+            _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=model,
-                data=data,
+                model=record.model,
+                data=record.data,
             )
             raise SearchError(
-                message=message, data=data, model=model, issue_id=issue.id
+                message=message, data=record.data, model=record.model, issue_id=issue.pk
             ) from err
 
-        queryset = transform_map.target_model.model_class().objects
+        return record
 
-        object = None
+    def get_model_or_update(self, record: DataRecord) -> ModelTypeVar | None:
+        # First check if there are any previous errors linked to this object
+        error_serial = self.get_error_serial(record.context, record.data)
+        if error_serial and error_serial in self.error_serials:
+            self.logger.log_info(
+                f"Skipping syncing of `{record.model}` with serial `{error_serial}` due to previous errors.",
+                obj=self.sync,
+            )
+            return None
+        record = self.get_transform_context(record)
+        queryset = record.transform_map.target_model.model_class().objects
+        model_settings = self.settings.get(record.model, False)
+
+        obj = None
         try:
             connection_name = self.get_db_connection_name()
             if model_settings:
-                logger.info(f"Creating {model}")
-                object = transform_map.update_or_create_instance(
-                    context=context,
+                logger.info(f"Creating {record.model}")
+                obj = record.transform_map.update_or_create_instance(
+                    context=record.context,
                     tags=self.sync.tags.all(),
                     connection_name=connection_name,
                 )
             else:
-                logger.info(f"Getting {model}")
-                context.pop("defaults", None)
-                object = queryset.using(connection_name).get(**context)
+                logger.info(f"Getting {record.model}")
+                record.context.pop("defaults", None)
+                obj = queryset.using(connection_name).get(**record.context)
         except queryset.model.DoesNotExist as err:
-            message = f"Instance of `{model}` not found."
-            _, issue = create_or_get_sync_issue(
+            message = f"Instance of `{record.model}` not found."
+            _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=model,
-                context=context,
-                data=data,
+                model=record.model,
+                context=record.context,
+                data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=model,
-                context=context,
-                data=data,
-                issue_id=issue.id,
+                model=record.model,
+                context=record.context,
+                data=record.data,
+                issue_id=issue.pk,
             ) from err
         except queryset.model.MultipleObjectsReturned as err:
-            message = f"Multiple instances of `{model}` found."
-            _, issue = create_or_get_sync_issue(
+            message = f"Multiple instances of `{record.model}` found."
+            _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
                 message=message,
-                model=model,
-                context=context,
-                data=data,
+                model=record.model,
+                context=record.context,
+                data=record.data,
             )
             raise SearchError(
                 message=message,
-                model=model,
-                context=context,
-                data=data,
-                issue_id=issue.id,
+                model=record.model,
+                context=record.context,
+                data=record.data,
+                issue_id=issue.pk,
             ) from err
         except Exception as err:
-            _, issue = create_or_get_sync_issue(
+            _, issue = self.create_or_get_sync_issue(
                 exception=err,
                 ingestion=self.ingestion,
-                model=model,
-                context=context,
-                data=data,
+                model=record.model,
+                context=record.context,
+                data=record.data,
             )
             raise SyncDataError(
-                model=model, context=context, data=data, issue_id=issue.id
+                model=record.model,
+                context=record.context,
+                data=record.data,
+                issue_id=issue.pk,
             ) from err
 
-        return object
+        return obj
 
     def collect_data(self):
         try:
@@ -298,6 +598,7 @@ class IPFabricSyncRunner(object):
             )
             data = {}
             if self.sync.snapshot_data.source.type == IPFabricSourceTypeChoices.REMOTE:
+                # This requires data already pushed to NetBox by user, does not connect to IPF directly
                 self.logger.log_info(
                     "Remote collector checking for snapshot data.", obj=self.sync
                 )
@@ -305,197 +606,144 @@ class IPFabricSyncRunner(object):
                     raise SyncError(
                         "No snapshot data available. This is a remote sync. Push data to NetBox first."
                     )
-                data["site"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(type="site").values_list(
-                        "data", flat=True
-                    )
-                )
-                data["device"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(type="device").values_list(
-                        "data", flat=True
-                    )
-                )
-                data["virtualchassis"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(
-                        type="virtualchassis"
-                    ).values_list("data", flat=True)
-                )
-                data["interface"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(
-                        type="interface"
-                    ).values_list("data", flat=True)
-                )
-                data["inventoryitem"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(
-                        type="inventoryitem"
-                    ).values_list("data", flat=True)
-                )
-                data["vlan"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(type="vlan").values_list(
-                        "data", flat=True
-                    )
-                )
-                data["vrf"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(type="vrf").values_list(
-                        "data", flat=True
-                    )
-                )
-                data["prefix"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(type="prefix").values_list(
-                        "data", flat=True
+                for endpoint in Endpoints:
+                    data[endpoint.name.lower()] = list(
+                        self.sync.snapshot_data.ipf_data.filter(
+                            type=endpoint.name.lower()
+                        ).values_list("data", flat=True)
                     )
-                )
-                data["ipaddress"] = list(
-                    self.sync.snapshot_data.ipf_data.filter(
-                        type="ipaddress"
-                    ).values_list("data", flat=True)
-                )
             else:
+                # This pulls data directly from IP Fabric instance
                 self.logger.log_info(
                     "Local collector being used for snapshot data.", obj=self.sync
                 )
-                excluded_vendors = ["aws", "azure"]
-
-                query_filter = {
-                    "and": [{"vendor": ["neq", vendor]} for vendor in excluded_vendors]
-                }
-
-                if ingestion_sites := self.settings.get("sites"):
-                    site_filter = {
-                        "or": [{"siteName": ["eq", site]} for site in ingestion_sites]
-                    }
-                    query_filter["and"].append(site_filter)
-
-                    self.logger.log_info(
-                        f"Creating site filter `{json.dumps(site_filter)}`",
-                        obj=self.sync,
-                    )
-                else:
-                    site_filter = {}
-
-                data["site"] = self.client.inventory.sites.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
-                )
-
-                data["device"] = self.client.inventory.devices.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=query_filter
-                )
-
-                data[
-                    "virtualchassis"
-                ] = self.client.technology.platforms.stacks_members.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
-                )
-
-                data["interface"] = self.client.inventory.interfaces.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
-                )
-
-                inventory_item_filter = {
-                    "and": [
-                        {"sn": ["empty", False]},
-                        {"name": ["empty", False]},
-                    ]
-                }
-                if site_filter:
-                    inventory_item_filter["and"].append(site_filter)
-
-                data["inventoryitem"] = self.client.inventory.pn.all(
+                endpoint_handler = EndpointHandler(
+                    self.client,
                     snapshot_id=self.settings["snapshot_id"],
-                    filters=inventory_item_filter,
-                )
-
-                data["vlan"] = self.client.technology.vlans.site_summary.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
-                )
-
-                data["vrf"] = self.client.technology.routing.vrf_detail.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
-                )
-
-                if site_filter:
-                    networks_filter = {
-                        "and": [site_filter, {"and": [{"net": ["empty", False]}]}]
-                    }
-                else:
-                    networks_filter = {"and": [{"net": ["empty", False]}]}
-                self.logger.log_info(f"Creating network filter: `{networks_filter}`")
-                data["prefix"] = self.client.technology.managed_networks.networks.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=networks_filter
-                )
-
-                data[
-                    "ipaddress"
-                ] = self.client.technology.addressing.managed_ip_ipv4.all(
-                    snapshot_id=self.settings["snapshot_id"], filters=site_filter
                 )
+                ingestion_sites = self.settings.get("sites")
+                for endpoint in Endpoints:
+                    filters = Filters.get_filter(endpoint.name, ingestion_sites)
+                    logger.debug(
+                        f"Collecting data from endpoint: `{endpoint.value}` using filter `{json.dumps(filters)}`."
+                    )
+                    data[endpoint.name.lower()] = endpoint_handler.resolve_endpoint(
+                        endpoint.value
+                    )(
+                        filters=filters,
+                    )
+                    self.logger.log_info(
+                        f"Collected {len(data[endpoint.name.lower()])} items from endpoint `{endpoint.value}`.",
+                        obj=self.sync.snapshot_data.source,
+                    )
         except Exception as e:
             self.logger.log_failure(
                 f"Error collecting data from IP Fabric: {e}", obj=self.sync
             )
             raise SyncError(f"Error collecting data from IP Fabric: {e}")
+        return data
 
-        self.logger.log_info(
-            f"{len(data['site'])} sites collected", obj=self.sync.snapshot_data.source
-        )
-        self.logger.log_info(
-            f"{len(data['device'])} devices collected",
-            obj=self.sync.snapshot_data.source,
-        )
-        self.logger.log_info(
-            f"{len(data['virtualchassis'])} stack members collected",
-            obj=self.sync.snapshot_data.source,
+    @cache
+    def get_transform_map(self, app: str, model: str) -> "IPFabricTransformMap":
+        """Get transform map for given app and model. Cached to improve performance."""
+        return self.transform_maps.get(
+            target_model__app_label=app, target_model__model=model
         )
 
-        self.logger.log_info(
-            f"{len(data['interface'])} interfaces collected",
-            obj=self.sync.snapshot_data.source,
+    def create_new_data_record(self, app: str, model: str, data: dict) -> DataRecord:
+        """Extract only relevant source data according to transform map configuration."""
+        transform_map = self.get_transform_map(app=app, model=model)
+        try:
+            source_data = transform_map.strip_source_data(data)
+        except KeyError as err:
+            raise SyncError(
+                f"Missing key column {err.args[0]} in source data when preparing data for {app}_{model}."
+            ) from err
+        return DataRecord(
+            app=app, model=model, data=source_data, transform_map=transform_map
         )
 
-        self.logger.log_info(
-            f"{len(data.get('inventoryitem', []))} part numbers collected",
-            obj=self.sync.snapshot_data.source,
-        )
+    def preprocess_data(self, data: dict) -> dict[str, set[DataRecord]]:
+        # TODO: Only process data according to settings to improve performance
+        # Set those records that can't be iterated separately
+        # Others are as empty set to define order which is shown in UI progress
+        records = {
+            "site": set(
+                self.create_new_data_record(app="dcim", model="site", data=item)
+                for item in data.get("site", [])
+            ),
+            "manufacturer": set(),
+            "devicetype": set(),
+            "platform": set(),
+            "devicerole": set(),
+            "device": set(),
+            "virtualchassis": set(),
+            "interface": set(),
+            "macaddress": set(),
+            "inventoryitem": set(
+                self.create_new_data_record(
+                    app="dcim", model="inventoryitem", data=item
+                )
+                for item in data.get("inventoryitem", [])
+            ),
+            "vlan": set(
+                self.create_new_data_record(app="ipam", model="vlan", data=item)
+                for item in data.get("vlan", [])
+            ),
+            "vrf": set(
+                self.create_new_data_record(app="ipam", model="vrf", data=item)
+                for item in data.get("vrf", [])
+            ),
+            "prefix": set(
+                self.create_new_data_record(app="ipam", model="prefix", data=item)
+                for item in data.get("prefix", [])
+            ),
+            "ipaddress": set(),
+        }
 
-        self.logger.log_info(
-            f"{len(data.get('vlan', []))} VLANs collected",
-            obj=self.sync.snapshot_data.source,
-        )
+        self.logger.log_info("Preparing devices", obj=self.sync)
+        members = order_members(data.get("virtualchassis", []))
+        devices, virtualchassis = order_devices(data.get("device", []), members)
 
-        self.logger.log_info(
-            f"{len(data.get('vrf', []))} VRFs collected",
-            obj=self.sync.snapshot_data.source,
-        )
+        # We need to store primary IPs of Devices to assign them later
+        # since they are not stored on Device object directly
+        # TODO: This will be later replaced when we are able to sync from multiple API tables to 1 model
+        device_primary_ips = {}
 
-        self.logger.log_info(
-            f"{len(data.get('prefix', []))} networks collected",
-            obj=self.sync.snapshot_data.source,
-        )
+        for device in devices:
+            records["manufacturer"].add(
+                self.create_new_data_record(
+                    app="dcim", model="manufacturer", data=device
+                )
+            )
+            records["devicetype"].add(
+                self.create_new_data_record(app="dcim", model="devicetype", data=device)
+            )
+            records["platform"].add(
+                self.create_new_data_record(app="dcim", model="platform", data=device)
+            )
+            records["devicerole"].add(
+                self.create_new_data_record(app="dcim", model="devicerole", data=device)
+            )
+            # This field is required by Device transform maps, but is set only when Device is part of VC.
+            if "virtual_chassis" not in device:
+                device["virtual_chassis"] = None
+            records["device"].add(
+                self.create_new_data_record(app="dcim", model="device", data=device)
+            )
+            device_primary_ips[device.get("sn")] = device.get("loginIp")
 
-        self.logger.log_info(
-            f"{len(data.get('ipaddress', []))} management IP's collected",
-            obj=self.sync.snapshot_data.source,
+        records["virtualchassis"] = set(
+            self.create_new_data_record(app="dcim", model="virtualchassis", data=item)
+            for item in virtualchassis
         )
-        self.logger.log_info("Ordering devices", obj=self.sync)
-
-        members = order_members(data.get("virtualchassis", []))
-        devices = order_devices(data.get("device", []), members)
-
-        self.logger.log_info("Ordering Part Numbers", obj=self.sync)
-
-        interface_dict = {}
-        for interface in data["interface"]:
-            if int_sn := interface.get("sn"):
-                if interface_dict.get(int_sn):
-                    interface_dict[int_sn].append(interface)
-                else:
-                    interface_dict[int_sn] = [interface]
 
+        # `nameOriginal` is human-readable interface name hidden column in IP Fabric
+        # This allows us to use it instead of the `intName`
+        # But it can be customized using transform maps, so we need to use the current value
         interface_key = "nameOriginal"
         try:
-            int_transform_map = self.transform_maps.get(
-                target_model__app_label="dcim", target_model__model="interface"
-            )
+            int_transform_map = self.get_transform_map(app="dcim", model="interface")
             int_name_field_map = int_transform_map.field_maps.get(target_field="name")
             interface_key = int_name_field_map.source_field
         except Exception as e:
@@ -505,71 +753,62 @@ class IPFabricSyncRunner(object):
             )
             raise SyncError(f"Error collecting source column name for interface: {e}")
 
-        managed_ips = {}
-        for ip in data["ipaddress"]:
-            # Find corresponding interface list by serial number (sn)
-            device_interfaces = interface_dict.get(ip["sn"], [])
+        self.logger.log_info("Preparing Interfaces", obj=self.sync)
+        # Store human-readable interface names to use them later for IP Addresses
+        readable_int_names = {}
+        for interface in data["interface"]:
+            interface_record = self.create_new_data_record(
+                app="dcim", model="interface", data=interface
+            )
+            interface_record.data["loginIp"] = device_primary_ips.get(
+                interface.get("sn")
+            )
+            records["interface"].add(interface_record)
+            readable_int_names[
+                f"{interface.get('sn')}_{interface.get('intName')}"
+            ] = interface.get(interface_key)
+            records["macaddress"].add(
+                self.create_new_data_record(
+                    app="dcim", model="macaddress", data=interface
+                )
+            )
 
-            # Use filter to find the interface with the matching intName
-            filtered_interface = list(
-                filter(lambda d: d["intName"] == ip["intName"], device_interfaces)
+        self.logger.log_info("Preparing IP Addresses", obj=self.sync)
+        for ip in data["ipaddress"]:
+            # We get `nameOriginal` from Interface table to get human-readable name instead fo `intName`
+            ip["nameOriginal"] = readable_int_names.get(
+                f"{ip.get('sn')}_{ip.get('intName')}"
+            )
+            # Let's skip IPs we cannot assign to an interface
+            if not ip["nameOriginal"]:
+                continue
+            ipaddress_record = self.create_new_data_record(
+                app="ipam", model="ipaddress", data=ip
             )
+            # Store whether this IP is primary for the device
+            ipaddress_record.data["is_primary"] = ip.get(
+                "sn"
+            ) in device_primary_ips and device_primary_ips.get(
+                ip.get("sn")
+            ) == ipaddress_record.data.get(
+                "ip"
+            )
+            records["ipaddress"].add(ipaddress_record)
 
-            if filtered_interface:
-                ip["nameOriginal"] = filtered_interface[0]["nameOriginal"]
-                if ip[interface_key]:
-                    int_name = ip[interface_key]
-                else:
-                    int_name = ip["intName"]
-                if ip["sn"] not in managed_ips:
-                    managed_ips[ip["sn"]] = {int_name: [ip]}
-                elif int_name not in managed_ips.get(ip["sn"]):
-                    managed_ips[ip["sn"]][int_name] = [ip]
-                else:
-                    managed_ips[ip["sn"]][int_name].append(ip)
-
-        for vlan in data["vlan"][:]:
-            # Remove VLANs with ID 0, minimum VLAN ID in NetBox is 1
-            if vlan.get("vlanId") == 0:
-                data["vlan"].remove(vlan)
-
-        for item in data["inventoryitem"][:]:
-            # Remove items with empty serial number
-            if item.get("sn") in [None, "None"]:
-                data["inventoryitem"].remove(item)
-
-        for model, item_count in [
-            ("site", len(data.get("site", []))),
-            ("device", len(devices)),
-            ("interface", len(data.get("interface", []))),
-            ("inventoryitem", len(data.get("inventoryitem", []))),
-            ("vlan", len(data.get("vlan", []))),
-            ("vrf", len(data.get("vrf", []))),
-            ("prefix", len(data.get("prefix", []))),
-            # TODO: Since we sync only those assigned to interfaces, we are skipping some IPs
-            # TODO: This is fixable by syncing IPs separately from interface and only assign them on interfaces
-            ("ipaddress", len(data.get("ipaddress", []))),
-        ]:
-            if self.settings.get(model):
-                self.logger.init_statistics(model, item_count)
+        for model, records_set in records.items():
+            if self.settings.get(model) and len(records_set):
+                self.logger.init_statistics(model, len(records_set))
+                self.logger.log_info(
+                    f"Prepared {len(records_set)} items for `{model}` to be synced.",
+                    obj=self.sync,
+                )
 
-        return (
-            data["site"],
-            devices,
-            interface_dict,
-            data["inventoryitem"],
-            data["vrf"],
-            data["vlan"],
-            data["prefix"],
-            managed_ips,
-        )
+        return records
 
     @handle_errors
     def sync_model(
         self,
-        app_label: str,
-        model: str,
-        data: dict | None,
+        record: DataRecord,
         stats: bool = True,
         sync: bool = False,
     ) -> ModelTypeVar | None:
@@ -578,31 +817,29 @@ class IPFabricSyncRunner(object):
         if not sync:
             return None
 
-        if not data:
+        if not record.data:
             return None
 
-        instance = self.get_model_or_update(app_label, model, data)
+        instance = self.get_model_or_update(record)
 
         # Only log when we successfully synced the item and asked for it
-        if stats:
-            self.logger.increment_statistics(model=model)
+        if stats and instance:
+            self.logger.increment_statistics(model=record.model)
 
         return instance
 
     def sync_item(
         self,
-        item,
-        app_label: str,
-        model: str,
+        record: DataRecord,
         cf: bool = False,
         ingestion: "IPFabricIngestion" = None,
+        stats: bool = True,
     ) -> ModelTypeVar | None:
         """Sync a single item to NetBox."""
         synced_object = self.sync_model(
-            app_label=app_label,
-            model=model,
-            data=item,
-            sync=self.settings.get(model),
+            record=record,
+            sync=self.settings.get(record.model),
+            stats=stats,
         )
         if synced_object is None:
             return None
@@ -620,13 +857,16 @@ class IPFabricSyncRunner(object):
 
     def sync_items(
         self,
-        items,
-        app_label: str,
-        model: str,
+        items: set[DataRecord],
         cf: bool = False,
         ingestion: "IPFabricIngestion" = None,
+        stats: bool = True,
     ) -> None:
         """Sync list of items to NetBox."""
+        if not items:
+            return
+
+        app, model = (lambda x: (x.app, x.model))(next(iter(items)))
         if not self.settings.get(model):
             self.logger.log_info(
                 f"Did not ask to sync {model}s, skipping.", obj=self.sync
@@ -634,230 +874,111 @@ class IPFabricSyncRunner(object):
             return
 
         for item in items:
-            self.sync_item(item, app_label, model, cf, ingestion)
+            self.sync_item(item, cf, ingestion, stats)
 
     @handle_errors
-    def sync_devices(
-        self,
-        ingestion,
-        devices,
-        interface_dict,
-        managed_ips,
-    ):
-        for model, name in [
-            ("manufacturer", "manufacturers"),
-            ("devicetype", "device types"),
-            ("platform", "platforms"),
-            ("devicerole", "device roles"),
-            ("virtualchassis", "virtual chassis"),
-            ("device", "devices"),
-            ("inventoryitem", "device inventory items"),
-        ]:
-            if not self.settings.get(model):
-                self.logger.log_info(
-                    f"Did not ask to sync {name}, skipping", obj=self.sync
-                )
-
-        devices_total = len(devices)
-
-        for device in devices:
-            self.sync_model(
-                "dcim", "manufacturer", device, sync=self.settings.get("manufacturer")
-            )
-            self.sync_model(
-                "dcim", "devicetype", device, sync=self.settings.get("devicetype")
-            )
-            self.sync_model(
-                "dcim", "platform", device, sync=self.settings.get("platform")
-            )
-            self.sync_model(
-                "dcim", "devicerole", device, sync=self.settings.get("devicerole")
-            )
-
-            virtual_chassis = device.get("virtual_chassis", {})
-            self.sync_model(
-                "dcim",
-                "virtualchassis",
-                virtual_chassis,
-                stats=False,
-                sync=self.settings.get("virtualchassis"),
-            )
-
-            # We need to get a Device instance even when not syncing it but syncing Interfaces, IPs or MACs
-            device_object: Device | None = self.sync_model(
-                "dcim",
-                "device",
-                device,
-                stats=False,
-                sync=self.settings.get("device")
-                or self.settings.get("interface")
-                or self.settings.get("ipaddress")
-                or self.settings.get("macaddress"),
+    def sync_ip_addresses(self, ip_addresses: set[DataRecord]):
+        """
+        We cannot assign primary IP in signals since IPAddress does not
+        contain information whether it is primary or not. And it must be done
+        on Device object, so cannot be done via Transform Maps yet since that
+        would require another Transform Map for Device.
+        So we need to do it manually here.
+        """
+        if not self.settings.get("ipaddress"):
+            self.logger.log_info(
+                "Did not ask to sync ipaddresses, skipping.", obj=self.sync
             )
+            return
 
-            if device_object and self.settings.get("device"):
-                device_object.snapshot()
-                if self.sync.update_custom_fields:
-                    device_object.custom_field_data[
-                        "ipfabric_source"
-                    ] = self.sync.snapshot_data.source.pk
-                    if ingestion:
-                        device_object.custom_field_data[
-                            "ipfabric_ingestion"
-                        ] = ingestion.pk
-                device_object.save()
-
-                self.logger.increment_statistics(model="device")
-                logger.info(
-                    f"Device {self.logger.log_data.get('statistics', {}).get('device', {}).get('current')} out of {devices_total}"
-                )
+        for ip_address in ip_addresses:
+            connection_name = self.get_db_connection_name()
 
-                # The Device exists now, so we can update the master of the VC.
-                # The logic is handled in transform maps.
-                self.sync_model(
-                    "dcim",
-                    "virtualchassis",
-                    virtual_chassis,
-                    stats=False,
-                    sync=self.settings.get("virtualchassis"),
+            # First remove primary IP from the target object.
+            # It cannot be done using hooks since there is no pre_clean at it fails on full_clean()
+            try:
+                ipv4_address = render_jinja2(
+                    ip_address.transform_map.field_maps.get(
+                        target_field="address"
+                    ).template,
+                    {"object": ip_address.data},
                 )
-
-            device_interfaces = interface_dict.get(device.get("sn"), [])
-            for device_interface in device_interfaces:
-                self.sync_interface(
-                    device_interface, managed_ips, device_object, device
+                other_device = (
+                    Device.objects.using(connection_name)
+                    .exclude(serial=serial(ip_address.data))
+                    .get(primary_ip4__address=ipv4_address)
                 )
-
-    @handle_errors
-    def sync_ipaddress(
-        self,
-        managed_ip: dict | None,
-        device_object: Device | None,
-        primary_ip: str | None,
-        login_ip: str | None,
-    ):
-        ip_address_obj: "IPAddress | None" = self.sync_model(
-            "ipam",
-            "ipaddress",
-            managed_ip,
-            sync=self.settings.get("ipaddress"),
-        )
-        if ip_address_obj is None:
-            return None
-
-        connection_name = self.get_db_connection_name()
-
-        try:
-            # Removing another IP is done in .signals.clear_other_primary_ip
-            # But do it here too, so the change is shown in StagedChange diff
-            other_device = Device.objects.using(connection_name).get(
-                primary_ip4=ip_address_obj
-            )
-            if other_device and device_object != other_device:
                 other_device.snapshot()
                 other_device.primary_ip4 = None
                 other_device.save(using=connection_name)
-        except ObjectDoesNotExist:
-            pass
-
-        if login_ip == primary_ip:
-            try:
-                device_object.snapshot()
-                device_object.primary_ip4 = ip_address_obj
-                device_object.save(using=connection_name)
-            except (ValueError, AttributeError) as err:
-                self.logger.log_failure(
-                    f"Error assigning primary IP to device: {err}", obj=self.sync
+            except Device.DoesNotExist:
+                # There is no other device with this IP as primary, all good
+                pass
+            except Exception as err:
+                # The transform maps might be changed, and we fail to resolve the template
+                # Make sure this does not crash the sync and is handled gracefully
+                _, issue = self.create_or_get_sync_issue(
+                    exception=err,
+                    ingestion=self.ingestion,
+                    message="Error removing primary IP from other device.",
+                    model=ip_address.model,
+                    data=ip_address.data,
                 )
-                return None
-        return ip_address_obj
-
-    @handle_errors
-    def sync_macaddress(
-        self, data: dict | None, interface_object: Interface
-    ) -> MACAddress | None:
-        # Need to create MAC Address object before we can assign it to Interface
-        # TODO: Figure out how to do this using transform maps
-        macaddress_data = {
-            "mac": data,
-            "id": getattr(interface_object, "pk", None),
-        }
-        macaddress_object: MACAddress | None = self.sync_model(
-            "dcim", "macaddress", macaddress_data, sync=self.settings.get("macaddress")
-        )
-        if macaddress_object is None:
-            return None
-        try:
-            interface_object.snapshot()
-            interface_object.primary_mac_address = macaddress_object
-            interface_object.save(using=self.get_db_connection_name())
-        except ValueError as err:
-            self.logger.log_failure(
-                f"Error assigning MAC Address to interface: {err}", obj=self.sync
-            )
-            return None
-        return macaddress_object
-
-    @handle_errors
-    def sync_interface(
-        self,
-        device_interface: dict,
-        managed_ips: dict,
-        device_object: Device | None,
-        device: dict,
-    ):
-        device_interface["loginIp"] = device.get("loginIp")
-        # We need to get an Interface instance even when not syncing it but syncing IPs or MACs
-        interface_object: Interface | None = self.sync_model(
-            "dcim",
-            "interface",
-            device_interface,
-            sync=self.settings.get("interface")
-            or self.settings.get("ipaddress")
-            or self.settings.get("macaddress"),
-        )
-
-        for ipaddress in managed_ips.get(
-            getattr(device_object, "serial", None), {}
-        ).get(getattr(interface_object, "name", None), []):
-            self.sync_ipaddress(
-                ipaddress,
-                device_object,
-                device_interface.get("primaryIp"),
-                device.get("loginIp"),
-            )
-
-        self.sync_macaddress(device_interface.get("mac"), interface_object)
-
-        return interface_object
+                raise IPAddressPrimaryRemovalError(
+                    data=ip_address.data,
+                    model=ip_address.model,
+                    issue_id=issue.pk,
+                ) from err
+
+            ip_address_obj: "IPAddress | None" = self.sync_item(record=ip_address)
+            if ip_address_obj is None or ip_address_obj.assigned_object is None:
+                continue
+
+            parent_device = ip_address_obj.assigned_object.parent_object
+
+            # Now assign this IP as primary to the parent device, if not assigned yet or assigned to different IP
+            if ip_address.data.get("is_primary") and (
+                not parent_device.primary_ip4
+                or parent_device.primary_ip4.pk != ip_address_obj.pk
+            ):
+                try:
+                    parent_device.snapshot()
+                    parent_device.primary_ip4 = ip_address_obj
+                    parent_device.save(using=connection_name)
+                except (ValueError, AttributeError) as err:
+                    self.logger.log_failure(
+                        f"Error assigning primary IP to device: {err}", obj=self.sync
+                    )
 
     def collect_and_sync(self, ingestion=None) -> None:
-        self.logger.log_info("Starting data sync.", obj=self.sync)
-        (
-            sites,
-            devices,
-            interface_dict,
-            inventory_items,
-            vrfs,
-            vlans,
-            networks,
-            managed_ips,
-        ) = self.collect_data()
+        self.logger.log_info("Starting data collection.", obj=self.sync)
+        data = self.collect_data()
+        self.logger.log_info("Starting to prepare items.", obj=self.sync)
+        records = self.preprocess_data(data=data)
 
+        self.logger.log_info("Starting data sync.", obj=self.sync)
         self.sync_items(
-            app_label="dcim",
-            model="site",
-            items=sites,
+            items=records["site"],
             cf=self.sync.update_custom_fields,
             ingestion=ingestion,
         )
-        self.sync_devices(
-            ingestion,
-            devices,
-            interface_dict,
-            managed_ips,
+        self.sync_items(items=records["manufacturer"])
+        self.sync_items(items=records["devicetype"])
+        self.sync_items(items=records["platform"])
+        self.sync_items(items=records["devicerole"])
+        self.sync_items(items=records["virtualchassis"])
+        self.sync_items(
+            items=records["device"],
+            cf=self.sync.update_custom_fields,
+            ingestion=ingestion,
         )
-        self.sync_items(app_label="dcim", model="inventoryitem", items=inventory_items)
-        self.sync_items(app_label="ipam", model="vlan", items=vlans)
-        self.sync_items(app_label="ipam", model="vrf", items=vrfs)
-        self.sync_items(app_label="ipam", model="prefix", items=networks)
+        # The Device exists now, so we can update the master of the VC.
+        # The logic is handled in transform maps.
+        self.sync_items(items=records["virtualchassis"], stats=False)
+        self.sync_items(items=records["interface"])
+        self.sync_items(items=records["macaddress"])
+        self.sync_items(items=records["inventoryitem"])
+        self.sync_items(items=records["vlan"])
+        self.sync_items(items=records["vrf"])
+        self.sync_items(items=records["prefix"])
+        self.sync_ip_addresses(ip_addresses=records["ipaddress"])