annet 2.3.1__tar.gz → 2.5.0__tar.gz

{annet-2.3.1/annet.egg-info → annet-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: annet
-Version: 2.3.1
+Version: 2.5.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -21,9 +21,10 @@ Requires-Dist: yarl>=1.8.2
 Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.2.1; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.2.2; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python

annet-2.5.0/annet/adapters/netbox/common/adapter.py

@@ -0,0 +1,60 @@
+from abc import abstractmethod, ABC
+from typing import Protocol, Generic, TypeVar
+
+from annet.annlib.netdev.views.hardware import HardwareView
+from .manufacturer import get_breed, get_hw
+from .models import NetboxDevice, Interface, IpAddress, Prefix
+
+NetboxDeviceT = TypeVar("NetboxDeviceT", bound=NetboxDevice)
+InterfaceT = TypeVar("InterfaceT", bound=Interface)
+IpAddressT = TypeVar("IpAddressT", bound=IpAddress)
+PrefixT = TypeVar("PrefixT", bound=Prefix)
+
+
+def get_device_breed(device: NetboxDeviceT) -> str:
+    if device.device_type and device.device_type.manufacturer:
+        return get_breed(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        )
+    return ""
+
+
+def get_device_hw(device: NetboxDeviceT) -> HardwareView:
+    if device.device_type and device.device_type.manufacturer:
+        return get_hw(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+            device.platform.name if device.platform else "",
+        )
+    return HardwareView("", "")
+
+
+class NetboxAdapter(ABC, Generic[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT]):
+    @abstractmethod
+    def list_all_fqdns(self) -> list[str]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_device(self, device_id: int) -> NetboxDeviceT:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces_by_devices(self, device_ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces(self, ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipaddr_by_ifaces(self, iface_ids: list[int]) -> list[IpAddressT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixT]:
+        raise NotImplementedError()

annet-2.5.0/annet/adapters/netbox/common/models.py

@@ -0,0 +1,287 @@
+from abc import abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from ipaddress import ip_interface, IPv6Interface
+from typing import List, Optional, Any, Dict, Sequence, TypeVar, Generic
+
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
+from annet.storage import Storage
+
+
+@dataclass
+class Entity(DumpableView):
+    id: int
+    name: str
+
+    @property
+    def _dump__list_key(self):
+        return self.name
+
+
+@dataclass
+class Label:
+    value: str
+    label: str
+
+
+@dataclass
+class IpFamily:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceType:
+    id: int
+    manufacturer: Entity
+    model: str
+
+
+@dataclass
+class DeviceIp(DumpableView):
+    id: int
+    display: str
+    address: str
+    family: int
+
+    @property
+    def _dump__list_key(self):
+        return self.address
+
+
+@dataclass
+class Prefix(DumpableView):
+    id: int
+    prefix: str
+    # `site` deprecated since v4.2, replace in derived classes.
+    vrf: Optional[Entity]
+    tenant: Optional[Entity]
+    vlan: Optional[Entity]
+    role: Optional[Entity]
+    status: Label
+    is_pool: bool
+    custom_fields: dict[str, Any]
+    created: datetime
+    last_updated: datetime
+    description: Optional[str] = ""
+
+    @property
+    def _dump__list_key(self):
+        return self.prefix
+
+
+_PrefixT = TypeVar("_PrefixT", bound=Prefix)
+
+
+@dataclass
+class IpAddress(DumpableView, Generic[_PrefixT]):
+    id: int
+    assigned_object_id: int | None
+    display: str
+    family: IpFamily
+    address: str
+    status: Label
+    tags: List[Entity]
+    created: datetime
+    last_updated: datetime
+    prefix: Optional[_PrefixT] = None
+    vrf: Optional[Entity] = None
+
+    @property
+    def _dump__list_key(self):
+        return self.address
+
+
+@dataclass
+class InterfaceConnectedEndpoint(Entity):
+    device: Entity
+
+
+@dataclass
+class InterfaceType:
+    value: str
+    label: str
+
+
+@dataclass
+class InterfaceMode:
+    value: str
+    label: str
+
+
+@dataclass
+class InterfaceVlan(Entity):
+    vid: int
+
+
+def vrf_object(vrf: str | None) -> Entity | None:
+    if vrf is None:
+        return None
+    else:
+        return Entity(id=0, name=vrf)
+
+
+_IpAddressT = TypeVar("_IpAddressT", bound=IpAddress)
+
+
+@dataclass
+class Interface(Entity, Generic[_IpAddressT]):
+    device: Entity
+    enabled: bool
+    description: str
+    type: InterfaceType
+    connected_endpoints: Optional[list[InterfaceConnectedEndpoint]]
+    mode: Optional[InterfaceMode]
+    untagged_vlan: Optional[InterfaceVlan]
+    tagged_vlans: Optional[List[InterfaceVlan]]
+    display: str = ""
+    ip_addresses: List[_IpAddressT] = field(default_factory=list)
+    vrf: Optional[Entity] = None
+    mtu: int | None = None
+    lag: Entity | None = None
+    lag_min_links: int | None = None
+
+    def add_addr(self, address_mask: str, vrf: str | None) -> None:
+        for existing_addr in self.ip_addresses:
+            if existing_addr.address == address_mask and (
+                    (existing_addr.vrf is None and vrf is None) or
+                    (existing_addr.vrf is not None and existing_addr.vrf.name == vrf)
+            ):
+                return
+
+        addr = ip_interface(address_mask)
+        vrf_obj = vrf_object(vrf)
+        if isinstance(addr, IPv6Interface):
+            family = IpFamily(value=6, label="IPv6")
+        else:
+            family = IpFamily(value=4, label="IPv4")
+        self._add_new_addr(address_mask, vrf_obj, family)
+
+    @abstractmethod
+    def _add_new_addr(self, address_mask: str, vrf: Entity | None, family: IpFamily):
+        raise NotImplementedError
+
+
+_InterfaceT = TypeVar("_InterfaceT", bound=Interface)
+
+
+@dataclass
+class NetboxDevice(Entity, Generic[_InterfaceT]):
+    url: str
+    storage: Storage
+
+    display: str
+    device_type: DeviceType
+    # `device_role` deprecated since v4.0, replace in derived classes.
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: Dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+    fqdn: str
+    hostname: str
+    hw: Optional[HardwareView]
+    breed: str
+
+    interfaces: List[_InterfaceT]
+
+    @property
+    def neighbors(self) -> List["Entity"]:
+        return [
+            endpoint.device
+            for iface in self.interfaces
+            if iface.connected_endpoints
+            for endpoint in iface.connected_endpoints
+            if endpoint.device
+        ]
+
+    # compat
+    @property
+    def neighbours_fqdns(self) -> list[str]:
+        return [dev.name for dev in self.neighbors]
+
+    @property
+    def neighbours_ids(self):
+        return [dev.id for dev in self.neighbors]
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.url == other.url
+
+    def is_pc(self) -> bool:
+        custom_breed_pc = ("Mellanox", "NVIDIA", "Moxa", "Nebius")
+        return self.device_type.manufacturer.name in custom_breed_pc or self.breed == "pc"
+
+    @abstractmethod
+    def _make_interface(self, name: str, type: InterfaceType) -> _InterfaceT:
+        raise NotImplementedError
+
+    def _lag_name(self, lag: int) -> str:
+        return lag_name(self.hw, lag)
+
+    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: int | None) -> _InterfaceT:
+        new_name = self._lag_name(lag)
+        for target_interface in self.interfaces:
+            if target_interface.name == new_name:
+                return target_interface
+        lag_interface = self._make_interface(
+            name=new_name,
+            type=InterfaceType(value="lag", label="Link Aggregation Group (LAG)"),
+        )
+        lag_interface.lag_min_links = lag_min_links
+        for interface in self.interfaces:
+            if interface.name in ports:
+                interface.lag = lag_interface
+        self.interfaces.append(lag_interface)
+        return lag_interface
+
+    def _svi_name(self, svi: int) -> str:
+        return svi_name(self.hw, svi)
+
+    def add_svi(self, svi: int) -> _InterfaceT:
+        name = self._svi_name(svi)
+        for interface in self.interfaces:
+            if interface.name == name:
+                return interface
+        interface = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(interface)
+        return interface
+
+    def _subif_name(self, interface: str, subif: int) -> str:
+        return f"{interface}.{subif}"
+
+    def add_subif(self, interface: str, subif: int) -> _InterfaceT:
+        name = self._subif_name(interface, subif)
+        for target_port in self.interfaces:
+            if target_port.name == name:
+                return target_port
+        target_port = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(target_port)
+        return target_port
+
+    def find_interface(self, name: str) -> Optional[_InterfaceT]:
+        for iface in self.interfaces:
+            if iface.name == name:
+                return iface
+        return None

{annet-2.3.1 → annet-2.5.0}/annet/adapters/netbox/common/status_client.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 from typing import Dict
+import re
 
 from adaptix import Retort, name_mapping, NameStyle
 from dataclass_rest import get
@@ -13,6 +14,12 @@ class Status:
     netbox_version: str
     plugins: Dict[str, str]
 
+    @property
+    def minor_version(self) -> str:
+        if match := re.match(r"\d+\.\d+", self.netbox_version):
+            return match.group(0)
+        return ""
+
 
 class NetboxStatusClient(BaseNetboxClient):
     def _init_response_body_factory(self) -> FactoryProtocol:

annet-2.3.1/annet/adapters/netbox/v37/storage.py → annet-2.5.0/annet/adapters/netbox/common/storage_base.py

@@ -1,88 +1,36 @@
 import ssl
-from collections import defaultdict
 from ipaddress import ip_interface
 from logging import getLogger
-from typing import Any, Optional, List, Union, Dict, cast
+from typing import Any, Optional, List, Union, Dict, cast, Generic, TypeVar
 
-from adaptix import P
-from adaptix.conversion import impl_converter, link, link_constant
 from annetbox.v37 import models as api_models
-from annetbox.v37.client_sync import NetboxV37
 
-from annet.adapters.netbox.common import models
-from annet.adapters.netbox.common.manufacturer import (
-    get_hw, get_breed,
-)
 from annet.adapters.netbox.common.query import NetboxQuery, FIELD_VALUE_SEPARATOR
 from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
-from annet.annlib.netdev.views.hardware import HardwareView
-from annet.storage import Storage, Device, Interface
+from annet.storage import Storage
+from .adapter import NetboxAdapter
+from .models import IpAddress, Interface, NetboxDevice, Prefix
 
 logger = getLogger(__name__)
+NetboxDeviceT = TypeVar("NetboxDeviceT", bound=NetboxDevice)
+InterfaceT = TypeVar("InterfaceT", bound=Interface)
+IpAddressT = TypeVar("IpAddressT", bound=IpAddress)
+PrefixT = TypeVar("PrefixT", bound=Prefix)
+
+
+class BaseNetboxStorage(
+    Storage,
+    Generic[
+        NetboxDeviceT,
+        InterfaceT,
+        IpAddressT,
+        PrefixT,
+    ],
+):
+    """
+    Base class for Netbox storage
+    """
 
-
-@impl_converter(recipe=[
-    link(P[api_models.Device].name, P[models.NetboxDevice].hostname),
-    link(P[api_models.Device].name, P[models.NetboxDevice].fqdn),
-])
-def extend_device_base(
-        device: api_models.Device,
-        interfaces: List[models.Interface],
-        hw: Optional[HardwareView],
-        breed: str,
-        storage: Storage,
-) -> models.NetboxDevice:
-    ...
-
-
-def extend_device(
-        device: api_models.Device,
-        interfaces: List[models.Interface],
-        storage: Storage,
-) -> models.NetboxDevice:
-    platform_name: str = ""
-    breed: str = ""
-    hw = HardwareView("", "")
-    if device.platform:
-        platform_name = device.platform.name
-    if device.device_type and device.device_type.manufacturer:
-        breed = get_breed(
-            device.device_type.manufacturer.name,
-            device.device_type.model,
-        )
-        hw = get_hw(
-            device.device_type.manufacturer.name,
-            device.device_type.model,
-            platform_name,
-        )
-    res = extend_device_base(
-        device=device,
-        interfaces=interfaces,
-        breed=breed,
-        hw=hw,
-        storage=storage,
-    )
-    return res
-
-
-@impl_converter(
-    recipe=[link_constant(P[models.Interface].lag_min_links, value=None)],
-)
-def extend_interface(
-        interface: api_models.Interface,
-        ip_addresses: List[models.IpAddress],
-) -> models.Interface:
-    ...
-
-
-@impl_converter
-def extend_ip_address(
-        ip_address: models.IpAddress, prefix: Optional[models.Prefix],
-) -> models.IpAddress:
-    ...
-
-
-class NetboxStorageV37(Storage):
     def __init__(self, opts: Optional[NetboxStorageOpts] = None):
         ctx: Optional[ssl.SSLContext] = None
         url = ""
@@ -98,12 +46,20 @@ class NetboxStorageV37(Storage):
             token = opts.token
             threads = opts.threads
             self.exact_host_filter = opts.exact_host_filter
-
-        self.netbox = NetboxV37(url=url, token=token, ssl_context=ctx, threads=threads)
+        self.netbox = self._init_adapter(url=url, token=token, ssl_context=ctx, threads=threads)
         self._all_fqdns: Optional[list[str]] = None
-        self._id_devices: dict[int, models.NetboxDevice] = {}
-        self._name_devices: dict[str, models.NetboxDevice] = {}
-        self._short_name_devices: dict[str, models.NetboxDevice] = {}
+        self._id_devices: dict[int, NetboxDeviceT] = {}
+        self._name_devices: dict[str, NetboxDeviceT] = {}
+        self._short_name_devices: dict[str, NetboxDeviceT] = {}
+
+    def _init_adapter(
+            self,
+            url: str,
+            token: str,
+            ssl_context: Optional[ssl.SSLContext],
+            threads: int,
+    ) -> NetboxAdapter[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT]:
+        raise NotImplementedError()
 
     def __enter__(self):
         return self
@@ -123,10 +79,7 @@ class NetboxStorageV37(Storage):
 
     def resolve_all_fdnds(self) -> list[str]:
         if self._all_fqdns is None:
-            self._all_fqdns = [
-                d.name
-                for d in self.netbox.dcim_all_devices_brief().results
-            ]
+            self._all_fqdns = self.netbox.list_all_fqdns()
         return self._all_fqdns
 
     def make_devices(
@@ -136,7 +89,7 @@ class NetboxStorageV37(Storage):
             use_mesh=None,
             preload_extra_fields=False,
             **kwargs,
-    ) -> List[models.NetboxDevice]:
+    ) -> List[NetboxDeviceT]:
         if isinstance(query, list):
             query = NetboxQuery.new(query)
 
@@ -154,106 +107,71 @@ class NetboxStorageV37(Storage):
                 return devices
             query = NetboxQuery.new(globs)
 
-        return devices + self._make_devices(
-            query=query,
-            preload_neighbors=preload_neighbors,
-            use_mesh=use_mesh,
-            preload_extra_fields=preload_extra_fields,
-            **kwargs
-        )
-
-    def _make_devices(
-            self,
-            query: NetboxQuery,
-            preload_neighbors=False,
-            use_mesh=None,
-            preload_extra_fields=False,
-            **kwargs,
-    ) -> List[models.NetboxDevice]:
-        device_ids = {
-            device.id: extend_device(
-                device=device,
-                interfaces=[],
-                storage=self,
-            )
-            for device in self._load_devices(query)
-        }
-        if not device_ids:
-            return []
-
-        for device in device_ids.values():
+        new_devices = self._load_devices(query)
+        self._fill_device_interfaces(new_devices)
+        for device in new_devices:
             self._record_device(device)
+        return devices + new_devices
 
-        interfaces = self._load_interfaces(list(device_ids))
-        for interface in interfaces:
-            device_ids[interface.device.id].interfaces.append(interface)
-
-        return list(device_ids.values())
-
-    def _record_device(self, device: models.NetboxDevice):
-        self._id_devices[device.id] = device
-        self._short_name_devices[device.name] = device
-        if not self.exact_host_filter:
-            short_name = device.name.split(".")[0]
-            self._short_name_devices[short_name] = device
-
-    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
+    def _load_devices(self, query: NetboxQuery) -> List[NetboxDeviceT]:
         if not query.globs:
             return []
         query_groups = parse_glob(self.exact_host_filter, query)
-        return [
+        devices = [
             device
-            for device in self.netbox.dcim_all_devices(**query_groups).results
+            for device in self.netbox.list_devices(query_groups)
             if _match_query(self.exact_host_filter, query, device)
         ]
+        return devices
 
-    def _extend_interfaces(self, interfaces: List[models.Interface]) -> List[models.Interface]:
-        extended_ifaces = {
-            interface.id: extend_interface(interface, [])
-            for interface in interfaces
-        }
-
-        ips = self.netbox.ipam_all_ip_addresses(interface_id=list(extended_ifaces))
-        ip_to_cidrs: Dict[str, str] = {ip.address: str(ip_interface(ip.address).network) for ip in ips.results}
-        prefixes = self.netbox.ipam_all_prefixes(prefix=list(ip_to_cidrs.values()))
-        cidr_to_prefix: Dict[str, models.Prefix] = {x.prefix: x for x in prefixes.results}
-
-        for ip in ips.results:
+    def _fill_device_interfaces(self, devices: list[NetboxDeviceT]) -> None:
+        device_mapping = {d.id: d for d in devices}
+        interfaces = self.netbox.list_interfaces_by_devices(list(device_mapping))
+        for interface in interfaces:
+            device_mapping[interface.device.id].interfaces.append(interface)
+        self._fill_interface_ipaddress(interfaces)
+
+    def _fill_interface_ipaddress(self, interfaces: list[InterfaceT]) -> None:
+        interface_mapping = {i.id: i for i in interfaces}
+        ips = self.netbox.list_ipaddr_by_ifaces(list(interface_mapping))
+        for ip in ips:
+            interface_mapping[ip.assigned_object_id].ip_addresses.append(ip)
+        self._fill_ipaddr_prefixes(ips)
+
+    def _fill_ipaddr_prefixes(self, ips: list[IpAddressT]) -> None:
+        ip_to_cidrs: Dict[str, str] = {ip.address: str(ip_interface(ip.address).network) for ip in ips}
+        prefixes = self.netbox.list_ipprefixes(list(ip_to_cidrs.values()))
+        cidr_to_prefix: Dict[str, PrefixT] = {x.prefix: x for x in prefixes}
+        for ip in ips:
             cidr = ip_to_cidrs[ip.address]
-            ip = extend_ip_address(ip, prefix=cidr_to_prefix.get(cidr))
-            extended_ifaces[ip.assigned_object_id].ip_addresses.append(ip)
-        return list(extended_ifaces.values())
+            ip.prefix = cidr_to_prefix.get(cidr)
 
-    def _load_interfaces(self, device_ids: List[int]) -> List[models.Interface]:
-        interfaces = self.netbox.dcim_all_interfaces(device_id=device_ids)
-        return self._extend_interfaces(interfaces.results)
-
-    def _load_interfaces_by_id(self, ids: List[int]) -> List[models.Interface]:
-        interfaces = self.netbox.dcim_all_interfaces_by_id(id=ids)
-        return self._extend_interfaces(interfaces.results)
+    def _record_device(self, device: NetboxDeviceT):
+        self._id_devices[device.id] = device
+        self._short_name_devices[device.name] = device
+        if not self.exact_host_filter:
+            short_name = device.name.split(".")[0]
+            self._short_name_devices[short_name] = device
 
     def get_device(
             self, obj_id, preload_neighbors=False, use_mesh=None,
             **kwargs,
-    ) -> models.NetboxDevice:
+    ) -> NetboxDeviceT:
         if obj_id in self._id_devices:
             return self._id_devices[obj_id]
 
-        device = self.netbox.dcim_device(obj_id)
-        interfaces = self._load_interfaces([device.id])
-
-        res = extend_device(
-            device=device,
-            storage=self,
-            interfaces=interfaces,
-        )
-        self._record_device(res)
-        return res
+        device = self.netbox.get_device(obj_id)
+        self._record_device(device)
+        return device
 
     def flush_perf(self):
         pass
 
-    def search_connections(self, device: Device, neighbor: Device) -> list[tuple[Interface, Interface]]:
+    def search_connections(
+            self,
+            device: NetboxDeviceT,
+            neighbor: NetboxDeviceT,
+    ) -> list[tuple[InterfaceT, InterfaceT]]:
         if device.storage is not self:
             raise ValueError("device does not belong to this storage")
         if neighbor.storage is not self: