annet 2.4.0__py3-none-any.whl → 2.5.1__py3-none-any.whl

annet/adapters/netbox/v37/storage.py

@@ -1,321 +1,91 @@
 import ssl
-from collections import defaultdict
-from ipaddress import ip_interface
-from logging import getLogger
-from typing import Any, Optional, List, Union, Dict, cast
 
 from adaptix import P
-from adaptix.conversion import impl_converter, link, link_constant
-from annetbox.v37 import models as api_models
-from annetbox.v37.client_sync import NetboxV37
+from adaptix.conversion import get_converter, link_function, link_constant, link
+from annetbox.v37 import models as api_models, client_sync
 
-from annet.adapters.netbox.common import models
-from annet.adapters.netbox.common.manufacturer import (
-    get_hw, get_breed,
-)
-from annet.adapters.netbox.common.query import NetboxQuery, FIELD_VALUE_SEPARATOR
-from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
-from annet.annlib.netdev.views.hardware import HardwareView
-from annet.storage import Storage, Device, Interface
+from annet.adapters.netbox.common.adapter import NetboxAdapter, get_device_breed, get_device_hw
+from annet.adapters.netbox.common.storage_base import BaseNetboxStorage
+from annet.storage import Storage
+from .models import IpAddressV37, NetboxDeviceV37, InterfaceV37, PrefixV37
 
-logger = getLogger(__name__)
 
-
-@impl_converter(recipe=[
-    link(P[api_models.Device].name, P[models.NetboxDevice].hostname),
-    link(P[api_models.Device].name, P[models.NetboxDevice].fqdn),
-])
-def extend_device_base(
-        device: api_models.Device,
-        interfaces: List[models.Interface],
-        hw: Optional[HardwareView],
-        breed: str,
-        storage: Storage,
-) -> models.NetboxDevice:
-    ...
-
-
-def extend_device(
-        device: api_models.Device,
-        interfaces: List[models.Interface],
-        storage: Storage,
-) -> models.NetboxDevice:
-    platform_name: str = ""
-    breed: str = ""
-    hw = HardwareView("", "")
-    if device.platform:
-        platform_name = device.platform.name
-    if device.device_type and device.device_type.manufacturer:
-        breed = get_breed(
-            device.device_type.manufacturer.name,
-            device.device_type.model,
+class NetboxV37Adapter(NetboxAdapter[NetboxDeviceV37, InterfaceV37, IpAddressV37, PrefixV37]):
+    def __init__(
+            self,
+            storage: Storage,
+            url: str,
+            token: str,
+            ssl_context: ssl.SSLContext | None,
+            threads: int,
+    ):
+        self.netbox = client_sync.NetboxV37(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.convert_device = get_converter(
+            api_models.Device,
+            NetboxDeviceV37,
+            recipe=[
+                link_function(get_device_breed, P[NetboxDeviceV37].breed),
+                link_function(get_device_hw, P[NetboxDeviceV37].hw),
+                link_constant(P[NetboxDeviceV37].interfaces, factory=list),
+                link_constant(P[NetboxDeviceV37].storage, value=storage),
+                link(P[api_models.Device].name, P[NetboxDeviceV37].hostname),
+                link(P[api_models.Device].name, P[NetboxDeviceV37].fqdn),
+            ]
         )
-        hw = get_hw(
-            device.device_type.manufacturer.name,
-            device.device_type.model,
-            platform_name,
+        self.convert_interfaces = get_converter(
+            list[api_models.Interface],
+            list[InterfaceV37],
+            recipe=[
+                link_constant(P[InterfaceV37].ip_addresses, factory=list),
+                link_constant(P[InterfaceV37].lag_min_links, value=None),
+            ]
         )
-    res = extend_device_base(
-        device=device,
-        interfaces=interfaces,
-        breed=breed,
-        hw=hw,
-        storage=storage,
-    )
-    return res
-
-
-@impl_converter(
-    recipe=[link_constant(P[models.Interface].lag_min_links, value=None)],
-)
-def extend_interface(
-        interface: api_models.Interface,
-        ip_addresses: List[models.IpAddress],
-) -> models.Interface:
-    ...
-
-
-@impl_converter
-def extend_ip_address(
-        ip_address: models.IpAddress, prefix: Optional[models.Prefix],
-) -> models.IpAddress:
-    ...
-
-
-class NetboxStorageV37(Storage):
-    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
-        ctx: Optional[ssl.SSLContext] = None
-        url = ""
-        token = ""
-        self.exact_host_filter = False
-        threads = 1
-        if opts:
-            if opts.insecure:
-                ctx = ssl.create_default_context()
-                ctx.check_hostname = False
-                ctx.verify_mode = ssl.CERT_NONE
-            url = opts.url
-            token = opts.token
-            threads = opts.threads
-            self.exact_host_filter = opts.exact_host_filter
-
-        self.netbox = NetboxV37(url=url, token=token, ssl_context=ctx, threads=threads)
-        self._all_fqdns: Optional[list[str]] = None
-        self._id_devices: dict[int, models.NetboxDevice] = {}
-        self._name_devices: dict[str, models.NetboxDevice] = {}
-        self._short_name_devices: dict[str, models.NetboxDevice] = {}
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, _, __, ___):
-        pass
-
-    def resolve_object_ids_by_query(self, query: NetboxQuery):
-        return [
-            d.id for d in self._load_devices(query)
-        ]
-
-    def resolve_fdnds_by_query(self, query: NetboxQuery):
-        return [
-            d.name for d in self._load_devices(query)
-        ]
-
-    def resolve_all_fdnds(self) -> list[str]:
-        if self._all_fqdns is None:
-            self._all_fqdns = [
-                d.name
-                for d in self.netbox.dcim_all_devices_brief().results
+        self.convert_ip_addresses = get_converter(
+            list[api_models.IpAddress],
+            list[IpAddressV37],
+            recipe=[
+                link_constant(P[IpAddressV37].prefix, value=None),
             ]
-        return self._all_fqdns
-
-    def make_devices(
-            self,
-            query: Union[NetboxQuery, list],
-            preload_neighbors=False,
-            use_mesh=None,
-            preload_extra_fields=False,
-            **kwargs,
-    ) -> List[models.NetboxDevice]:
-        if isinstance(query, list):
-            query = NetboxQuery.new(query)
-
-        devices = []
-        if query.is_host_query():
-            globs = []
-            for glob in query.globs:
-                if glob in self._name_devices:
-                    devices.append(self._name_devices[glob])
-                if glob in self._short_name_devices:
-                    devices.append(self._short_name_devices[glob])
-                else:
-                    globs.append(glob)
-            if not globs:
-                return devices
-            query = NetboxQuery.new(globs)
-
-        return devices + self._make_devices(
-            query=query,
-            preload_neighbors=preload_neighbors,
-            use_mesh=use_mesh,
-            preload_extra_fields=preload_extra_fields,
-            **kwargs
+        )
+        self.convert_ip_prefixes = get_converter(
+            list[api_models.Prefix],
+            list[PrefixV37],
         )
 
-    def _make_devices(
-            self,
-            query: NetboxQuery,
-            preload_neighbors=False,
-            use_mesh=None,
-            preload_extra_fields=False,
-            **kwargs,
-    ) -> List[models.NetboxDevice]:
-        device_ids = {
-            device.id: extend_device(
-                device=device,
-                interfaces=[],
-                storage=self,
-            )
-            for device in self._load_devices(query)
-        }
-        if not device_ids:
-            return []
-
-        for device in device_ids.values():
-            self._record_device(device)
-
-        interfaces = self._load_interfaces(list(device_ids))
-        for interface in interfaces:
-            device_ids[interface.device.id].interfaces.append(interface)
-
-        return list(device_ids.values())
-
-    def _record_device(self, device: models.NetboxDevice):
-        self._id_devices[device.id] = device
-        self._short_name_devices[device.name] = device
-        if not self.exact_host_filter:
-            short_name = device.name.split(".")[0]
-            self._short_name_devices[short_name] = device
-
-    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
-        if not query.globs:
-            return []
-        query_groups = parse_glob(self.exact_host_filter, query)
+    def list_all_fqdns(self) -> list[str]:
         return [
-            device
-            for device in self.netbox.dcim_all_devices(**query_groups).results
-            if _match_query(self.exact_host_filter, query, device)
+            d.name
+            for d in self.netbox.dcim_all_devices_brief().results
         ]
 
-    def _extend_interfaces(self, interfaces: List[models.Interface]) -> List[models.Interface]:
-        extended_ifaces = {
-            interface.id: extend_interface(interface, [])
-            for interface in interfaces
-        }
-
-        ips = self.netbox.ipam_all_ip_addresses(interface_id=list(extended_ifaces))
-        ip_to_cidrs: Dict[str, str] = {ip.address: str(ip_interface(ip.address).network) for ip in ips.results}
-        prefixes = self.netbox.ipam_all_prefixes(prefix=list(ip_to_cidrs.values()))
-        cidr_to_prefix: Dict[str, models.Prefix] = {x.prefix: x for x in prefixes.results}
-
-        for ip in ips.results:
-            cidr = ip_to_cidrs[ip.address]
-            ip = extend_ip_address(ip, prefix=cidr_to_prefix.get(cidr))
-            extended_ifaces[ip.assigned_object_id].ip_addresses.append(ip)
-        return list(extended_ifaces.values())
-
-    def _load_interfaces(self, device_ids: List[int]) -> List[models.Interface]:
-        interfaces = self.netbox.dcim_all_interfaces(device_id=device_ids)
-        return self._extend_interfaces(interfaces.results)
-
-    def _load_interfaces_by_id(self, ids: List[int]) -> List[models.Interface]:
-        interfaces = self.netbox.dcim_all_interfaces_by_id(id=ids)
-        return self._extend_interfaces(interfaces.results)
-
-    def get_device(
-            self, obj_id, preload_neighbors=False, use_mesh=None,
-            **kwargs,
-    ) -> models.NetboxDevice:
-        if obj_id in self._id_devices:
-            return self._id_devices[obj_id]
-
-        device = self.netbox.dcim_device(obj_id)
-        interfaces = self._load_interfaces([device.id])
-
-        res = extend_device(
-            device=device,
-            storage=self,
-            interfaces=interfaces,
-        )
-        self._record_device(res)
-        return res
-
-    def flush_perf(self):
-        pass
-
-    def search_connections(self, device: Device, neighbor: Device) -> list[tuple[Interface, Interface]]:
-        if device.storage is not self:
-            raise ValueError("device does not belong to this storage")
-        if neighbor.storage is not self:
-            raise ValueError("neighbor does not belong to this storage")
-        # both devices are NetboxDevice if they are loaded from this storage
-        res = []
-        for local_port in device.interfaces:
-            if not local_port.connected_endpoints:
-                continue
-            for endpoint in local_port.connected_endpoints:
-                if endpoint.device.id == neighbor.id:
-                    for remote_port in neighbor.interfaces:
-                        if remote_port.name == endpoint.name:
-                            res.append((local_port, remote_port))
-                            break
-        return res
-
-
-def _match_query(exact_host_filter: bool, query: NetboxQuery, device_data: api_models.Device) -> bool:
-    """
-    Additional filtering after netbox due to limited backend logic.
-    """
-    if exact_host_filter:
-        return True  # nothing to check, all filtering is done by netbox
-    hostnames = [subquery.strip() for subquery in query.globs if FIELD_VALUE_SEPARATOR not in subquery]
-    if not hostnames:
-        return True  # no hostnames to check
+    def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceV37]:
+        return [
+            self.convert_device(dev)
+            for dev in self.netbox.dcim_all_devices(**query).results
+        ]
 
-    short_name = device_data.name.split(".")[0]
-    for hostname in hostnames:
-        hostname = hostname.strip().rstrip(".")
-        if short_name == hostname or device_data.name == hostname:
-            return True
-    return False
+    def get_device(self, device_id: int) -> NetboxDeviceV37:
+        return self.convert_device(self.netbox.dcim_device(device_id))
 
+    def list_interfaces_by_devices(self, device_ids: list[int]) -> list[InterfaceV37]:
+        return self.convert_interfaces(self.netbox.dcim_all_interfaces(device_id=device_ids).results)
 
-def _hostname_dot_hack(raw_query: str) -> str:
-    # there is no proper way to lookup host by its hostname
-    # ie find "host" with fqdn "host.example.com"
-    # besides using name__ic (ie startswith)
-    # since there is no direct analogue for this field in netbox
-    # so we need to add a dot to hostnames (top-level fqdn part)
-    # so we would not receive devices with a common name prefix
-    def add_dot(raw_query: Any) -> Any:
-        if isinstance(raw_query, str) and "." not in raw_query:
-            raw_query = raw_query + "."
-        return raw_query
+    def list_interfaces(self, ids: list[int]) -> list[InterfaceV37]:
+        return self.convert_interfaces(self.netbox.dcim_all_interfaces(id=ids).results)
 
-    if isinstance(raw_query, list):
-        for i, name in enumerate(raw_query):
-            raw_query[i] = add_dot(name)
-    elif isinstance(raw_query, str):
-        raw_query = add_dot(raw_query)
+    def list_ipaddr_by_ifaces(self, iface_ids: list[int]) -> list[IpAddressV37]:
+        return self.convert_ip_addresses(self.netbox.ipam_all_ip_addresses(interface_id=iface_ids).results)
 
-    return raw_query
+    def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixV37]:
+        return self.convert_ip_prefixes(self.netbox.ipam_all_prefixes(prefix=prefixes).results)
 
 
-def parse_glob(exact_host_filter: bool, query: NetboxQuery) -> dict[str, list[str]]:
-    query_groups = cast(dict[str, list[str]], query.parse_query())
-    if names := query_groups.pop("name", None):
-        if exact_host_filter:
-            query_groups["name__ie"] = names
-        else:
-            query_groups["name__ic"] = [_hostname_dot_hack(name) for name in names]
-    return query_groups
+class NetboxStorageV37(BaseNetboxStorage[NetboxDeviceV37, InterfaceV37, IpAddressV37, PrefixV37]):
+    def _init_adapter(
+            self,
+            url: str,
+            token: str,
+            ssl_context: ssl.SSLContext | None,
+            threads: int,
+    ) -> NetboxAdapter[NetboxDeviceV37, InterfaceV37, IpAddressV37, PrefixV37]:
+        return NetboxV37Adapter(self, url, token, ssl_context, threads)

annet/adapters/netbox/v41/models.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+from typing import Optional
+import warnings
+from datetime import datetime, timezone
+from annet.adapters.netbox.common.models import Entity, Interface, InterfaceType, IpAddress, Label, NetboxDevice, DeviceIp, IpFamily, Prefix
+
+
+@dataclass
+class PrefixV41(Prefix):
+    site: Optional[Entity] = None
+
+
+@dataclass
+class IpAddressV41(IpAddress[PrefixV41]):
+    prefix: Optional[PrefixV41] = None
+
+
+@dataclass
+class InterfaceV41(Interface[IpAddressV41]):
+    def _add_new_addr(self, address_mask: str, vrf: Entity | None, family: IpFamily) -> None:
+        self.ip_addresses.append(IpAddressV41(
+            id=0,
+            display=address_mask,
+            address=address_mask,
+            vrf=vrf,
+            prefix=None,
+            family=family,
+            created=datetime.now(timezone.utc),
+            last_updated=datetime.now(timezone.utc),
+            tags=[],
+            status=Label(value="active", label="Active"),
+            assigned_object_id=self.id,
+        ))
+
+
+@dataclass
+class DeviceIpV41(DeviceIp):
+    id: int
+    display: str
+    address: str
+    family: IpFamily
+
+
+@dataclass
+class NetboxDeviceV41(NetboxDevice[InterfaceV41]):
+    role: Entity
+    primary_ip: Optional[DeviceIpV41]
+    primary_ip4: Optional[DeviceIpV41]
+    primary_ip6: Optional[DeviceIpV41]
+
+    @property
+    def device_role(self):
+        warnings.warn(
+            "'device_role' is deprecated, use 'role' instead.",
+            DeprecationWarning,
+            stacklevel=2
+        )
+        return self.role
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def _make_interface(self, name: str, type: InterfaceType) -> InterfaceV41:
+        return InterfaceV41(
+            name=name,
+            device=self,
+            enabled=True,
+            description="",
+            type=type,
+            id=0,
+            vrf=None,
+            display=name,
+            untagged_vlan=None,
+            tagged_vlans=[],
+            ip_addresses=[],
+            connected_endpoints=[],
+            mode=None,
+        )

annet/adapters/netbox/v41/storage.py

@@ -0,0 +1,91 @@
+import ssl
+from adaptix import P
+from adaptix.conversion import get_converter, link, link_constant, link_function
+from annetbox.v41 import client_sync
+from annetbox.v41 import models as api_models
+
+from annet.adapters.netbox.common.adapter import NetboxAdapter, get_device_breed, get_device_hw
+from annet.adapters.netbox.common.storage_base import BaseNetboxStorage
+from annet.adapters.netbox.v41.models import InterfaceV41, IpAddressV41, NetboxDeviceV41, PrefixV41
+from annet.storage import Storage
+
+
+class NetboxV41Adapter(NetboxAdapter[NetboxDeviceV41, InterfaceV41, IpAddressV41, PrefixV41]):
+    def __init__(
+            self,
+            storage: Storage,
+            url: str,
+            token: str,
+            ssl_context: ssl.SSLContext | None,
+            threads: int,
+    ):
+        self.netbox = client_sync.NetboxV41(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.convert_device = get_converter(
+            api_models.Device,
+            NetboxDeviceV41,
+            recipe=[
+                link_function(get_device_breed, P[NetboxDeviceV41].breed),
+                link_function(get_device_hw, P[NetboxDeviceV41].hw),
+                link_constant(P[NetboxDeviceV41].interfaces, factory=list),
+                link_constant(P[NetboxDeviceV41].storage, value=storage),
+                link(P[api_models.Device].name, P[NetboxDeviceV41].hostname),
+                link(P[api_models.Device].name, P[NetboxDeviceV41].fqdn),
+            ]
+        )
+        self.convert_interfaces = get_converter(
+            list[api_models.Interface],
+            list[InterfaceV41],
+            recipe=[
+                link_constant(P[InterfaceV41].ip_addresses, factory=list),
+                link_constant(P[InterfaceV41].lag_min_links, value=None),
+            ]
+        )
+        self.convert_ip_addresses = get_converter(
+            list[api_models.IpAddress],
+            list[IpAddressV41],
+            recipe=[
+                link_constant(P[IpAddressV41].prefix, value=None),
+            ]
+        )
+        self.convert_ip_prefixes = get_converter(
+            list[api_models.Prefix],
+            list[PrefixV41],
+        )
+
+    def list_all_fqdns(self) -> list[str]:
+        return [
+            d.name
+            for d in self.netbox.dcim_all_devices_brief().results
+        ]
+
+    def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceV41]:
+        return [
+            self.convert_device(dev)
+            for dev in self.netbox.dcim_all_devices(**query).results
+        ]
+
+    def get_device(self, device_id: int) -> NetboxDeviceV41:
+        return self.convert_device(self.netbox.dcim_device(device_id))
+
+    def list_interfaces_by_devices(self, device_ids: list[int]) -> list[InterfaceV41]:
+        return self.convert_interfaces(self.netbox.dcim_all_interfaces(device_id=device_ids).results)
+
+    def list_interfaces(self, ids: list[int]) -> list[InterfaceV41]:
+        return self.convert_interfaces(self.netbox.dcim_all_interfaces(id=ids).results)
+
+    def list_ipaddr_by_ifaces(self, iface_ids: list[int]) -> list[IpAddressV41]:
+        return self.convert_ip_addresses(self.netbox.ipam_all_ip_addresses(interface_id=iface_ids).results)
+
+    def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixV41]:
+        return self.convert_ip_prefixes(self.netbox.ipam_all_prefixes(prefix=prefixes).results)
+
+
+class NetboxStorageV41(BaseNetboxStorage[NetboxDeviceV41, InterfaceV41, IpAddressV41, PrefixV41]):
+    def _init_adapter(
+            self,
+            url: str,
+            token: str,
+            ssl_context: ssl.SSLContext | None,
+            threads: int,
+    ) -> NetboxAdapter[NetboxDeviceV41, InterfaceV41, IpAddressV41, PrefixV41]:
+        return NetboxV41Adapter(self, url, token, ssl_context, threads)

annet/adapters/netbox/v42/models.py

@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Optional
+from annet.adapters.netbox.common.models import InterfaceType, IpFamily, Label, Prefix, Entity
+from annet.adapters.netbox.v41.models import InterfaceV41, IpAddressV41, NetboxDeviceV41
+
+
+@dataclass
+class PrefixV42(Prefix):
+    scope: Optional[Entity] = None
+    scope_type: str | None = None
+
+    @property
+    def site(self) -> Optional[Entity]:
+        if self.scope_type == "dcim.site":
+            return self.scope
+        return None
+
+
+@dataclass
+class IpAddressV42(IpAddressV41):
+    prefix: Optional[PrefixV42] = None
+
+
+@dataclass
+class InterfaceV42(InterfaceV41):
+    def _add_new_addr(self, address_mask: str, vrf: Entity | None, family: IpFamily) -> None:
+        self.ip_addresses.append(IpAddressV42(
+            id=0,
+            display=address_mask,
+            address=address_mask,
+            vrf=vrf,
+            prefix=None,
+            family=family,
+            created=datetime.now(timezone.utc),
+            last_updated=datetime.now(timezone.utc),
+            tags=[],
+            status=Label(value="active", label="Active"),
+            assigned_object_id=self.id,
+        ))
+
+
+@dataclass
+class NetboxDeviceV42(NetboxDeviceV41):
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def _make_interface(self, name: str, type: InterfaceType) -> InterfaceV42:
+        return InterfaceV42(
+            name=name,
+            device=self,
+            enabled=True,
+            description="",
+            type=type,
+            id=0,
+            vrf=None,
+            display=name,
+            untagged_vlan=None,
+            tagged_vlans=[],
+            ip_addresses=[],
+            connected_endpoints=[],
+            mode=None,
+        )