annet 2.4.0__py3-none-any.whl → 2.5.1__py3-none-any.whl

annet/adapters/netbox/common/adapter.py

@@ -0,0 +1,60 @@
+from abc import abstractmethod, ABC
+from typing import Protocol, Generic, TypeVar
+
+from annet.annlib.netdev.views.hardware import HardwareView
+from .manufacturer import get_breed, get_hw
+from .models import NetboxDevice, Interface, IpAddress, Prefix
+
+NetboxDeviceT = TypeVar("NetboxDeviceT", bound=NetboxDevice)
+InterfaceT = TypeVar("InterfaceT", bound=Interface)
+IpAddressT = TypeVar("IpAddressT", bound=IpAddress)
+PrefixT = TypeVar("PrefixT", bound=Prefix)
+
+
+def get_device_breed(device: NetboxDeviceT) -> str:
+    if device.device_type and device.device_type.manufacturer:
+        return get_breed(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        )
+    return ""
+
+
+def get_device_hw(device: NetboxDeviceT) -> HardwareView:
+    if device.device_type and device.device_type.manufacturer:
+        return get_hw(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+            device.platform.name if device.platform else "",
+        )
+    return HardwareView("", "")
+
+
+class NetboxAdapter(ABC, Generic[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT]):
+    @abstractmethod
+    def list_all_fqdns(self) -> list[str]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_device(self, device_id: int) -> NetboxDeviceT:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces_by_devices(self, device_ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces(self, ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipaddr_by_ifaces(self, iface_ids: list[int]) -> list[IpAddressT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixT]:
+        raise NotImplementedError()

annet/adapters/netbox/common/models.py

@@ -1,7 +1,8 @@
+from abc import abstractmethod
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import datetime
 from ipaddress import ip_interface, IPv6Interface
-from typing import List, Optional, Any, Dict, Sequence, Callable
+from typing import List, Optional, Any, Dict, Sequence, TypeVar, Generic
 
 from annet.annlib.netdev.views.dump import DumpableView
 from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
@@ -53,7 +54,7 @@ class DeviceIp(DumpableView):
 class Prefix(DumpableView):
     id: int
     prefix: str
-    site: Optional[Entity]
+    # `site` deprecated since v4.2, replace in derived classes.
     vrf: Optional[Entity]
     tenant: Optional[Entity]
     vlan: Optional[Entity]
@@ -70,10 +71,13 @@ class Prefix(DumpableView):
         return self.prefix
 
 
+_PrefixT = TypeVar("_PrefixT", bound=Prefix)
+
+
 @dataclass
-class IpAddress(DumpableView):
+class IpAddress(DumpableView, Generic[_PrefixT]):
     id: int
-    assigned_object_id: int
+    assigned_object_id: int | None
     display: str
     family: IpFamily
     address: str
@@ -81,7 +85,7 @@ class IpAddress(DumpableView):
     tags: List[Entity]
     created: datetime
     last_updated: datetime
-    prefix: Optional[Prefix] = None
+    prefix: Optional[_PrefixT] = None
     vrf: Optional[Entity] = None
 
     @property
@@ -111,8 +115,18 @@ class InterfaceVlan(Entity):
     vid: int
 
 
+def vrf_object(vrf: str | None) -> Entity | None:
+    if vrf is None:
+        return None
+    else:
+        return Entity(id=0, name=vrf)
+
+
+_IpAddressT = TypeVar("_IpAddressT", bound=IpAddress)
+
+
 @dataclass
-class Interface(Entity):
+class Interface(Entity, Generic[_IpAddressT]):
     device: Entity
     enabled: bool
     description: str
@@ -122,53 +136,44 @@ class Interface(Entity):
     untagged_vlan: Optional[InterfaceVlan]
     tagged_vlans: Optional[List[InterfaceVlan]]
     display: str = ""
-    ip_addresses: List[IpAddress] = field(default_factory=list)
+    ip_addresses: List[_IpAddressT] = field(default_factory=list)
     vrf: Optional[Entity] = None
     mtu: int | None = None
     lag: Entity | None = None
     lag_min_links: int | None = None
 
     def add_addr(self, address_mask: str, vrf: str | None) -> None:
-        addr = ip_interface(address_mask)
-        if vrf is None:
-            vrf_obj = None
-        else:
-            vrf_obj = Entity(id=0, name=vrf)
+        for existing_addr in self.ip_addresses:
+            if existing_addr.address == address_mask and (
+                    (existing_addr.vrf is None and vrf is None) or
+                    (existing_addr.vrf is not None and existing_addr.vrf.name == vrf)
+            ):
+                return
 
+        addr = ip_interface(address_mask)
+        vrf_obj = vrf_object(vrf)
         if isinstance(addr, IPv6Interface):
             family = IpFamily(value=6, label="IPv6")
         else:
             family = IpFamily(value=4, label="IPv4")
+        self._add_new_addr(address_mask, vrf_obj, family)
 
-        for existing_addr in self.ip_addresses:
-            if existing_addr.address == address_mask and (
-                (existing_addr.vrf is None and vrf is None) or
-                (existing_addr.vrf is not None and existing_addr.vrf.name == vrf)
-            ):
-                return
-        self.ip_addresses.append(IpAddress(
-            id=0,
-            display=address_mask,
-            address=address_mask,
-            vrf=vrf_obj,
-            prefix=None,
-            family=family,
-            created=datetime.now(timezone.utc),
-            last_updated=datetime.now(timezone.utc),
-            tags=[],
-            status=Label(value="active", label="Active"),
-            assigned_object_id=self.id,
-        ))
+    @abstractmethod
+    def _add_new_addr(self, address_mask: str, vrf: Entity | None, family: IpFamily):
+        raise NotImplementedError
+
+
+_InterfaceT = TypeVar("_InterfaceT", bound=Interface)
 
 
 @dataclass
-class NetboxDevice(Entity):
+class NetboxDevice(Entity, Generic[_InterfaceT]):
     url: str
     storage: Storage
 
     display: str
     device_type: DeviceType
-    device_role: Entity
+    # `device_role` deprecated since v4.0, replace in derived classes.
     tenant: Optional[Entity]
     platform: Optional[Entity]
     serial: str
@@ -191,7 +196,7 @@ class NetboxDevice(Entity):
     hw: Optional[HardwareView]
     breed: str
 
-    interfaces: List[Interface]
+    interfaces: List[_InterfaceT]
 
     @property
     def neighbors(self) -> List["Entity"]:
@@ -222,27 +227,14 @@ class NetboxDevice(Entity):
         custom_breed_pc = ("Mellanox", "NVIDIA", "Moxa", "Nebius")
         return self.device_type.manufacturer.name in custom_breed_pc or self.breed == "pc"
 
-    def _make_interface(self, name: str, type: InterfaceType) -> Interface:
-        return Interface(
-            name=name,
-            device=self,
-            enabled=True,
-            description="",
-            type=type,
-            id=0,
-            vrf=None,
-            display=name,
-            untagged_vlan=None,
-            tagged_vlans=[],
-            ip_addresses=[],
-            connected_endpoints=[],
-            mode=None,
-        )
+    @abstractmethod
+    def _make_interface(self, name: str, type: InterfaceType) -> _InterfaceT:
+        raise NotImplementedError
 
     def _lag_name(self, lag: int) -> str:
         return lag_name(self.hw, lag)
 
-    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: int | None) -> Interface:
+    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: int | None) -> _InterfaceT:
         new_name = self._lag_name(lag)
         for target_interface in self.interfaces:
             if target_interface.name == new_name:
@@ -261,7 +253,7 @@ class NetboxDevice(Entity):
     def _svi_name(self, svi: int) -> str:
         return svi_name(self.hw, svi)
 
-    def add_svi(self, svi: int) -> Interface:
+    def add_svi(self, svi: int) -> _InterfaceT:
         name = self._svi_name(svi)
         for interface in self.interfaces:
             if interface.name == name:
@@ -276,7 +268,7 @@ class NetboxDevice(Entity):
     def _subif_name(self, interface: str, subif: int) -> str:
         return f"{interface}.{subif}"
 
-    def add_subif(self, interface: str, subif: int) -> Interface:
+    def add_subif(self, interface: str, subif: int) -> _InterfaceT:
         name = self._subif_name(interface, subif)
         for target_port in self.interfaces:
             if target_port.name == name:
@@ -288,7 +280,7 @@ class NetboxDevice(Entity):
         self.interfaces.append(target_port)
         return target_port
 
-    def find_interface(self, name: str) -> Optional[Interface]:
+    def find_interface(self, name: str) -> Optional[_InterfaceT]:
         for iface in self.interfaces:
             if iface.name == name:
                 return iface

annet/adapters/netbox/common/status_client.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from typing import Dict
+import re
 
 from adaptix import Retort, name_mapping, NameStyle
 from dataclass_rest import get
@@ -13,6 +14,12 @@ class Status:
     netbox_version: str
     plugins: Dict[str, str]
 
+    @property
+    def minor_version(self) -> str:
+        if match := re.match(r"\d+\.\d+", self.netbox_version):
+            return match.group(0)
+        return ""
+
 
 class NetboxStatusClient(BaseNetboxClient):
     def _init_response_body_factory(self) -> FactoryProtocol:

annet/adapters/netbox/common/storage_base.py

@@ -0,0 +1,239 @@
+import ssl
+from ipaddress import ip_interface
+from logging import getLogger
+from typing import Any, Optional, List, Union, Dict, cast, Generic, TypeVar
+
+from annetbox.v37 import models as api_models
+
+from annet.adapters.netbox.common.query import NetboxQuery, FIELD_VALUE_SEPARATOR
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.storage import Storage
+from .adapter import NetboxAdapter
+from .models import IpAddress, Interface, NetboxDevice, Prefix
+
+logger = getLogger(__name__)
+NetboxDeviceT = TypeVar("NetboxDeviceT", bound=NetboxDevice)
+InterfaceT = TypeVar("InterfaceT", bound=Interface)
+IpAddressT = TypeVar("IpAddressT", bound=IpAddress)
+PrefixT = TypeVar("PrefixT", bound=Prefix)
+
+
+class BaseNetboxStorage(
+    Storage,
+    Generic[
+        NetboxDeviceT,
+        InterfaceT,
+        IpAddressT,
+        PrefixT,
+    ],
+):
+    """
+    Base class for Netbox storage
+    """
+
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        ctx: Optional[ssl.SSLContext] = None
+        url = ""
+        token = ""
+        self.exact_host_filter = False
+        threads = 1
+        if opts:
+            if opts.insecure:
+                ctx = ssl.create_default_context()
+                ctx.check_hostname = False
+                ctx.verify_mode = ssl.CERT_NONE
+            url = opts.url
+            token = opts.token
+            threads = opts.threads
+            self.exact_host_filter = opts.exact_host_filter
+        self.netbox = self._init_adapter(url=url, token=token, ssl_context=ctx, threads=threads)
+        self._all_fqdns: Optional[list[str]] = None
+        self._id_devices: dict[int, NetboxDeviceT] = {}
+        self._name_devices: dict[str, NetboxDeviceT] = {}
+        self._short_name_devices: dict[str, NetboxDeviceT] = {}
+
+    def _init_adapter(
+            self,
+            url: str,
+            token: str,
+            ssl_context: Optional[ssl.SSLContext],
+            threads: int,
+    ) -> NetboxAdapter[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT]:
+        raise NotImplementedError()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: NetboxQuery):
+        return [
+            d.id for d in self._load_devices(query)
+        ]
+
+    def resolve_fdnds_by_query(self, query: NetboxQuery):
+        return [
+            d.name for d in self._load_devices(query)
+        ]
+
+    def resolve_all_fdnds(self) -> list[str]:
+        if self._all_fqdns is None:
+            self._all_fqdns = self.netbox.list_all_fqdns()
+        return self._all_fqdns
+
+    def make_devices(
+            self,
+            query: Union[NetboxQuery, list],
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> List[NetboxDeviceT]:
+        if isinstance(query, list):
+            query = NetboxQuery.new(query)
+
+        devices = []
+        if query.is_host_query():
+            globs = []
+            for glob in query.globs:
+                if glob in self._name_devices:
+                    devices.append(self._name_devices[glob])
+                if glob in self._short_name_devices:
+                    devices.append(self._short_name_devices[glob])
+                else:
+                    globs.append(glob)
+            if not globs:
+                return devices
+            query = NetboxQuery.new(globs)
+
+        new_devices = self._load_devices(query)
+        self._fill_device_interfaces(new_devices)
+        for device in new_devices:
+            self._record_device(device)
+        return devices + new_devices
+
+    def _load_devices(self, query: NetboxQuery) -> List[NetboxDeviceT]:
+        if not query.globs:
+            return []
+        query_groups = parse_glob(self.exact_host_filter, query)
+        devices = [
+            device
+            for device in self.netbox.list_devices(query_groups)
+            if _match_query(self.exact_host_filter, query, device)
+        ]
+        return devices
+
+    def _fill_device_interfaces(self, devices: list[NetboxDeviceT]) -> None:
+        device_mapping = {d.id: d for d in devices}
+        interfaces = self.netbox.list_interfaces_by_devices(list(device_mapping))
+        for interface in interfaces:
+            device_mapping[interface.device.id].interfaces.append(interface)
+        self._fill_interface_ipaddress(interfaces)
+
+    def _fill_interface_ipaddress(self, interfaces: list[InterfaceT]) -> None:
+        interface_mapping = {i.id: i for i in interfaces}
+        ips = self.netbox.list_ipaddr_by_ifaces(list(interface_mapping))
+        for ip in ips:
+            interface_mapping[ip.assigned_object_id].ip_addresses.append(ip)
+        self._fill_ipaddr_prefixes(ips)
+
+    def _fill_ipaddr_prefixes(self, ips: list[IpAddressT]) -> None:
+        ip_to_cidrs: Dict[str, str] = {ip.address: str(ip_interface(ip.address).network) for ip in ips}
+        prefixes = self.netbox.list_ipprefixes(list(ip_to_cidrs.values()))
+        cidr_to_prefix: Dict[str, PrefixT] = {x.prefix: x for x in prefixes}
+        for ip in ips:
+            cidr = ip_to_cidrs[ip.address]
+            ip.prefix = cidr_to_prefix.get(cidr)
+
+    def _record_device(self, device: NetboxDeviceT):
+        self._id_devices[device.id] = device
+        self._short_name_devices[device.name] = device
+        if not self.exact_host_filter:
+            short_name = device.name.split(".")[0]
+            self._short_name_devices[short_name] = device
+
+    def get_device(
+            self, obj_id, preload_neighbors=False, use_mesh=None,
+            **kwargs,
+    ) -> NetboxDeviceT:
+        if obj_id in self._id_devices:
+            return self._id_devices[obj_id]
+
+        device = self.netbox.get_device(obj_id)
+        self._record_device(device)
+        return device
+
+    def flush_perf(self):
+        pass
+
+    def search_connections(
+            self,
+            device: NetboxDeviceT,
+            neighbor: NetboxDeviceT,
+    ) -> list[tuple[InterfaceT, InterfaceT]]:
+        if device.storage is not self:
+            raise ValueError("device does not belong to this storage")
+        if neighbor.storage is not self:
+            raise ValueError("neighbor does not belong to this storage")
+        # both devices are NetboxDevice if they are loaded from this storage
+        res = []
+        for local_port in device.interfaces:
+            if not local_port.connected_endpoints:
+                continue
+            for endpoint in local_port.connected_endpoints:
+                if endpoint.device.id == neighbor.id:
+                    for remote_port in neighbor.interfaces:
+                        if remote_port.name == endpoint.name:
+                            res.append((local_port, remote_port))
+                            break
+        return res
+
+
+def _match_query(exact_host_filter: bool, query: NetboxQuery, device_data: api_models.Device) -> bool:
+    """
+    Additional filtering after netbox due to limited backend logic.
+    """
+    if exact_host_filter:
+        return True  # nothing to check, all filtering is done by netbox
+    hostnames = [subquery.strip() for subquery in query.globs if FIELD_VALUE_SEPARATOR not in subquery]
+    if not hostnames:
+        return True  # no hostnames to check
+
+    short_name = device_data.name.split(".")[0]
+    for hostname in hostnames:
+        hostname = hostname.strip().rstrip(".")
+        if short_name == hostname or device_data.name == hostname:
+            return True
+    return False
+
+
+def _hostname_dot_hack(raw_query: str) -> str:
+    # there is no proper way to lookup host by its hostname
+    # ie find "host" with fqdn "host.example.com"
+    # besides using name__ic (ie startswith)
+    # since there is no direct analogue for this field in netbox
+    # so we need to add a dot to hostnames (top-level fqdn part)
+    # so we would not receive devices with a common name prefix
+    def add_dot(raw_query: Any) -> Any:
+        if isinstance(raw_query, str) and "." not in raw_query:
+            raw_query = raw_query + "."
+        return raw_query
+
+    if isinstance(raw_query, list):
+        for i, name in enumerate(raw_query):
+            raw_query[i] = add_dot(name)
+    elif isinstance(raw_query, str):
+        raw_query = add_dot(raw_query)
+
+    return raw_query
+
+
+def parse_glob(exact_host_filter: bool, query: NetboxQuery) -> dict[str, list[str]]:
+    query_groups = cast(dict[str, list[str]], query.parse_query())
+    if names := query_groups.pop("name", None):
+        if exact_host_filter:
+            query_groups["name__ie"] = names
+        else:
+            query_groups["name__ic"] = [_hostname_dot_hack(name) for name in names]
+    return query_groups

annet/adapters/netbox/provider.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, Optional
 
-from dataclass_rest.exceptions import ClientError
+from dataclass_rest.exceptions import ClientError, ClientLibraryError
 
 from annet.storage import StorageProvider, Storage
 from annet.connectors import AdapterWithName, AdapterWithConfig, T
@@ -9,21 +9,36 @@ from .common.storage_opts import NetboxStorageOpts
 from .common.query import NetboxQuery
 from .v24.storage import NetboxStorageV24
 from .v37.storage import NetboxStorageV37
+from .v41.storage import NetboxStorageV41
+from .v42.storage import NetboxStorageV42
 
 
 def storage_factory(opts: NetboxStorageOpts) -> Storage:
     client = NetboxStatusClient(opts.url, opts.token, opts.insecure)
+    version_class_map = {
+        "3.4": NetboxStorageV37,
+        "3.7": NetboxStorageV37,
+        "4.0": NetboxStorageV41,
+        "4.1": NetboxStorageV41,
+        "4.2": NetboxStorageV42,
+    }
+
+    status = None
+
     try:
         status = client.status()
+        for version_prefix, storage_class in version_class_map.items():
+            if version_prefix == status.minor_version:
+                return storage_class(opts)
+
     except ClientError as e:
         if e.status_code == 404:
-            # old version do not support status reqeust
             return NetboxStorageV24(opts)
-        raise
-    if status.netbox_version.startswith("3."):
-        return NetboxStorageV37(opts)
-    else:
-        raise ValueError(f"Unsupported version: {status.netbox_version}")
+        else:
+            raise ValueError(f"Unsupported version: {status.netbox_version}")
+    except ClientLibraryError:
+        raise ValueError(f"Connection error: Unable to reach Netbox at URL: {opts.url}")
+    raise Exception(f"Unsupported version: {status.netbox_version}")
 
 
 class NetboxProvider(StorageProvider, AdapterWithName, AdapterWithConfig):