annet 0.6__tar.gz → 0.8__tar.gz

{annet-0.6/annet.egg-info → annet-0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: annet
-Version: 0.6
+Version: 0.8
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -21,3 +21,5 @@ Requires-Dist: contextlog>=1.1
 Requires-Dist: valkit>=0.1.4
 Requires-Dist: aiohttp>=3.8.4
 Requires-Dist: yarl>=1.8.2
+Requires-Dist: adaptix==3.0.0b2
+Requires-Dist: dataclass-rest==0.4

{annet-0.6 → annet-0.8}/README.md

@@ -13,6 +13,15 @@ Usage help can be obtained by calling ```annet -h``` or for a specific command,
 
 ## Overview
 
+## Configuration
+
+Provide `NETBOX_URL` and `NETBOX_TOKEN` environment variable to setup data source.
+
+```shell
+export NETBOX_URL="https://demo.netbox.dev"
+export NETBOX_TOKEN="1234567890abcdef01234567890abcdef0123456"
+```
+
 ### annet gen
 
 The annet_generators directory contains many files called generators.

annet-0.8/annet/adapters/netbox/common/client.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass
+from functools import wraps
+from typing import Generic, Optional, List, TypeVar, Callable
+
+from dataclass_rest.http.requests import RequestsClient
+from requests import Session
+
+Model = TypeVar("Model")
+
+
+@dataclass
+class PagingResponse(Generic[Model]):
+    next: Optional[str]
+    previous: Optional[str]
+    count: int
+    results: List[Model]
+
+
+Func = TypeVar("Func", bound=Callable)
+
+
+def _collect_by_pages(func: Func) -> Func:
+    """Collect all results using only pagination."""
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        kwargs.setdefault("offset", 0)
+        limit = kwargs.setdefault("limit", 100)
+        results = []
+        method = func.__get__(self, self.__class__)
+        has_next = True
+        while has_next:
+            page = method(*args, **kwargs)
+            kwargs["offset"] += limit
+            results.extend(page.results)
+            has_next = bool(page.next)
+        return PagingResponse(
+            previous=None,
+            next=None,
+            count=len(results),
+            results=results,
+        )
+
+    return wrapper
+
+
+# default batch size 100 is calculated to fit list of UUIDs in 4k URL length
+def collect(func: Func, field: str = "", batch_size: int = 100) -> Func:
+    """
+    Collect data from method iterating over pages and filter batches.
+
+    :param func: Method to call
+    :param field: Field which defines a filter split into batches
+    :param batch_size: Limit of values in `field` filter requested at a time
+    """
+    func = _collect_by_pages(func)
+    if not field:
+        return func
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        value = kwargs.get(field)
+        if not value:
+            return func(*args, **kwargs)
+
+        method = func.__get__(self, self.__class__)
+        results = []
+        for offset in range(0, len(value), batch_size):
+            kwargs[field] = value[offset:offset + batch_size]
+            page = method(*args, **kwargs)
+            results.extend(page.results)
+        return PagingResponse(
+            previous=None,
+            next=None,
+            count=len(results),
+            results=results,
+        )
+
+    return wrapper
+
+
+class BaseNetboxClient(RequestsClient):
+    def __init__(self, url: str, token: str):
+        url = url.rstrip("/") + "/api/"
+        session = Session()
+        if token:
+            session.headers["Authorization"] = f"Token {token}"
+        super().__init__(url, session)

annet-0.8/annet/adapters/netbox/common/manufacturer.py

@@ -0,0 +1,62 @@
+from logging import getLogger
+
+from annet.annlib.netdev.views.hardware import HardwareView
+
+logger = getLogger(__name__)
+
+_VENDORS = {
+    "cisco": "Cisco",
+    "catalyst": "Cisco Catalyst",
+    "nexus": "Cisco Nexus",
+    "huawei": "Huawei",
+    "juniper": "Juniper",
+    "arista": "Arista",
+    "pc": "PC",
+    "nokia": "Nokia",
+    "aruba": "Aruba",
+    "routeros": "RouterOS",
+    "ribbon": "Ribbon",
+}
+
+
+def _vendor_to_hw(vendor):
+    return HardwareView(_VENDORS.get(vendor.lower(), vendor), None)
+
+
+def get_hw(manufacturer: str, model: str):
+    # by some reason Netbox calls Mellanox SN as MSN, so we fix them here
+    if manufacturer == "Mellanox" and model.startswith("MSN"):
+        model = model.replace("MSN", "SN", 1)
+    hw = _vendor_to_hw(manufacturer + " " + model)
+    if not hw:
+        raise ValueError(f"unsupported manufacturer {manufacturer}")
+    return hw
+
+
+def get_breed(manufacturer: str, model: str):
+    if manufacturer == "Huawei" and model.startswith("CE"):
+        return "vrp85"
+    elif manufacturer == "Huawei" and model.startswith("NE"):
+        return "vrp85"
+    elif manufacturer == "Huawei":
+        return "vrp55"
+    elif manufacturer == "Mellanox":
+        return "cuml2"
+    elif manufacturer == "Juniper":
+        return "jun10"
+    elif manufacturer == "Cisco":
+        return "ios12"
+    elif manufacturer == "Adva":
+        return "adva8"
+    elif manufacturer == "Arista":
+        return "eos4"
+    raise ValueError(f"unsupported manufacturer {manufacturer}")
+
+
+def is_supported(manufacturer: str) -> bool:
+    if manufacturer not in (
+            "Huawei", "Mellanox", "Juniper", "Cisco", "Adva", "Arista",
+    ):
+        logger.warning("Unsupported manufacturer `%s`", manufacturer)
+        return False
+    return True

annet-0.8/annet/adapters/netbox/common/models.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional, Any
+
+from annet.annlib.netdev.views.hardware import HardwareView
+
+
+@dataclass
+class Entity:
+    id: int
+    name: str
+
+
+@dataclass
+class Label:
+    value: str
+    label: str
+
+
+@dataclass
+class IpFamily:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceType:
+    id: int
+    manufacturer: Entity
+    model: str
+
+
+@dataclass
+class DeviceIp:
+    id: int
+    display: str
+    address: str
+    family: int
+
+
+@dataclass
+class IpAddress:
+    id: int
+    assigned_object_id: int
+    display: str
+    family: IpFamily
+    address: str
+    status: Label
+    tags: List[Entity]
+    created: datetime
+    last_updated: datetime
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+    display: str = ""
+    ip_addresses: List[IpAddress] = field(default_factory=list)
+
+
+@dataclass
+class NetboxDevice(Entity):
+    neighbours_ids: List[int]
+
+    display: str
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+    fqdn: str
+    hostname: str
+    hw: Optional[HardwareView]
+    breed: str
+
+    interfaces: List[Interface]
+
+    # compat
+    def __hash__(self):
+        return hash(self.id)
+
+    def is_pc(self):
+        return self.device_type.manufacturer.name == "Mellanox"

annet-0.8/annet/adapters/netbox/common/query.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import List, Union, Iterable, Optional
+
+from annet.storage import Query
+
+
+@dataclass
+class NetboxQuery(Query):
+    query: List[str]
+
+    @classmethod
+    def new(
+            cls, query: Union[str, Iterable[str]],
+            hosts_range: Optional[slice] = None,
+    ) -> "NetboxQuery":
+        if hosts_range is not None:
+            raise ValueError("host_range is not supported")
+        return cls(query=list(query))
+
+    @property
+    def globs(self):
+        # We process every query host as a glob
+        return self.query

annet-0.8/annet/adapters/netbox/common/status_client.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+
+from adaptix import Retort, name_mapping, NameStyle
+from dataclass_rest import get
+from dataclass_rest.client_protocol import FactoryProtocol
+
+from .client import BaseNetboxClient
+
+
+@dataclass
+class Status:
+    netbox_version: str
+    plugins: dict[str, str]
+
+
+class NetboxStatusClient(BaseNetboxClient):
+    def _init_response_body_factory(self) -> FactoryProtocol:
+        return Retort(recipe=[
+            name_mapping(name_style=NameStyle.LOWER_KEBAB)
+        ])
+
+    @get("status")
+    def status(self) -> Status:
+        ...

annet-0.8/annet/adapters/netbox/common/storage_opts.py

@@ -0,0 +1,14 @@
+import os
+
+
+class NetboxStorageOpts:
+    def __init__(self, url: str, token: str):
+        self.url = url
+        self.token = token
+
+    @classmethod
+    def from_cli_opts(cls, cli_opts):
+        return cls(
+            url=os.getenv("NETBOX_URL", "http://localhost"),
+            token=os.getenv("NETBOX_TOKEN", "").strip(),
+        )

annet-0.8/annet/adapters/netbox/provider.py

@@ -0,0 +1,34 @@
+from dataclass_rest.exceptions import ClientError
+
+from annet.storage import StorageProvider, Storage
+from .common.status_client import NetboxStatusClient
+from .common.storage_opts import NetboxStorageOpts
+from .common.query import NetboxQuery
+from .v24.storage import NetboxStorageV24
+from .v37.storage import NetboxStorageV37
+
+
+def storage_factory(opts: NetboxStorageOpts) -> Storage:
+    client = NetboxStatusClient(opts.url, opts.token)
+    try:
+        status = client.status()
+    except ClientError as e:
+        if e.status_code == 404:
+            # old version do not support status reqeust
+            return NetboxStorageV24(opts)
+        raise
+    if status.netbox_version.startswith("3."):
+        return NetboxStorageV37(opts)
+    else:
+        raise ValueError(f"Unsupported version: {status.netbox_version}")
+
+
+class NetboxProvider(StorageProvider):
+    def storage(self):
+        return storage_factory
+
+    def opts(self):
+        return NetboxStorageOpts
+
+    def query(self):
+        return NetboxQuery

annet-0.8/annet/adapters/netbox/v24/api_models.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List, Optional, Any
+
+from annet.adapters.netbox.common.models import Entity, DeviceType
+
+
+@dataclass
+class Label:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceIp:
+    id: int
+    address: str
+    family: int
+
+
+@dataclass
+class Device(Entity):
+    display_name: str
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[str]
+    custom_fields: dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+
+
+@dataclass
+class Vrf(Entity):
+    rd: str
+
+
+@dataclass
+class IpAddress:
+    id: int
+    family: int
+    address: str
+    vrf: Optional[Vrf]
+    tenant: Any  # ???
+    status: Label
+    description: Optional[str]
+    custom_fields: dict[str, Any]
+    tags: List[str]
+    created: datetime
+    last_updated: datetime
+
+    interface: Entity
+
+    nat_inside: Any  # ???
+    nat_outside: Any  # ???

annet-0.8/annet/adapters/netbox/v24/client.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+from typing import List, Optional
+
+import dateutil.parser
+from adaptix import Retort, loader
+from dataclass_rest import get
+
+from annet.adapters.netbox.common.client import (
+    BaseNetboxClient, collect, PagingResponse,
+)
+from .api_models import Device, Interface, IpAddress
+
+
+class NetboxV24(BaseNetboxClient):
+    def _init_response_body_factory(self) -> Retort:
+        return Retort(recipe=[
+            loader(datetime, dateutil.parser.parse)
+        ])
+
+    @get("dcim/interfaces")
+    def interfaces(
+            self,
+            device_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Interface]:
+        pass
+
+    all_interfaces = collect(interfaces, field="device_id")
+
+    @get("ipam/ip-addresses")
+    def ip_addresses(
+            self,
+            interface_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[IpAddress]:
+        pass
+
+    all_ip_addresses = collect(ip_addresses, field="interface_id")
+
+    @get("dcim/devices")
+    def devices(
+            self,
+            name: Optional[List[str]] = None,
+            tag: Optional[str] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Device]:
+        pass
+
+    all_devices = collect(devices)
+
+    @get("dcim/devices/{device_id}")
+    def get_device(
+            self,
+            device_id: int,
+    ) -> Device:
+        pass

annet-0.8/annet/adapters/netbox/v24/storage.py

@@ -0,0 +1,190 @@
+from logging import getLogger
+from typing import Optional, List
+
+from annet.adapters.netbox.common import models
+from annet.adapters.netbox.common.manufacturer import (
+    is_supported, get_hw, get_breed,
+)
+from annet.adapters.netbox.common.query import NetboxQuery
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.storage import Storage
+from . import api_models
+from .client import NetboxV24
+
+logger = getLogger(__name__)
+
+
+def extend_device_ip(
+        ip: Optional[api_models.DeviceIp],
+) -> Optional[models.DeviceIp]:
+    if not ip:
+        return None
+    return models.DeviceIp(
+        address=ip.address,
+        id=ip.id,
+        display=ip.address,
+        family=ip.family,
+    )
+
+
+def extend_label(
+        label: Optional[api_models.Label],
+) -> Optional[models.Label]:
+    if not label:
+        return None
+    return models.Label(
+        label=label.label,
+        value=str(label.value),
+    )
+
+
+def extend_device(
+        device: api_models.Device,
+) -> models.NetboxDevice:
+    manufacturer = device.device_type.manufacturer.name
+    model = device.device_type.model
+
+    return models.NetboxDevice(
+        id=device.id,
+        name=device.name,
+        display=device.display_name,
+        device_type=device.device_type,
+        device_role=device.device_role,
+        tenant=device.tenant,
+        platform=device.platform,
+        serial=device.serial,
+        asset_tag=device.asset_tag,
+        site=device.site,
+        rack=device.rack,
+        position=device.position,
+        face=extend_label(device.face),
+        status=device.status,
+        primary_ip=extend_device_ip(device.primary_ip),
+        primary_ip4=extend_device_ip(device.primary_ip4),
+        primary_ip6=extend_device_ip(device.primary_ip6),
+        tags=[models.Entity(0, tag) for tag in device.tags],
+        custom_fields=device.custom_fields,  # ???
+        created=device.created,
+        last_updated=device.last_updated,
+
+        fqdn=device.name,
+        hostname=device.name,
+        hw=get_hw(manufacturer, model),
+        breed=get_breed(manufacturer, model),
+        interfaces=[],
+        neighbours_ids=[],
+    )
+
+
+def extend_interface(interface: api_models.Interface) -> models.Interface:
+    return models.Interface(
+        id=interface.id,
+        name=interface.name,
+        device=interface.device,
+        enabled=interface.enabled,
+        display=interface.name,
+        ip_addresses=[],
+    )
+
+
+def extend_ip(ip: api_models.IpAddress) -> models.IpAddress:
+    return models.IpAddress(
+        id=ip.id,
+        assigned_object_id=ip.interface.id,
+        display=ip.address,
+        family=models.IpFamily(
+            value=ip.family,
+            label=str(ip.family),
+        ),
+        address=ip.address,
+        status=extend_label(ip.status),
+        tags=[models.Entity(0, tag) for tag in ip.tags],
+        created=ip.created,
+        last_updated=ip.last_updated,
+    )
+
+
+class NetboxStorageV24(Storage):
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        self.netbox = NetboxV24(
+            url=opts.url,
+            token=opts.token,
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: NetboxQuery):
+        return [
+            d.id for d in self._load_devices(query)
+        ]
+
+    def resolve_fdnds_by_query(self, query: NetboxQuery):
+        return [
+            d.name for d in self._load_devices(query)
+        ]
+
+    def make_devices(
+            self,
+            query: NetboxQuery,
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> list[models.NetboxDevice]:
+        device_ids = {
+            device.id: extend_device(device=device)
+            for device in self._load_devices(query)
+        }
+        if not device_ids:
+            return []
+
+        interfaces = self._load_interfaces(list(device_ids))
+        for interface in interfaces:
+            device_ids[interface.device.id].interfaces.append(interface)
+        return list(device_ids.values())
+
+    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
+        return [
+            device
+            for device in self.netbox.all_devices().results
+            if _match_query(query, device)
+            if is_supported(device.device_type.manufacturer.name)
+        ]
+
+    def _load_interfaces(self, device_ids: List[int]) -> List[
+        models.Interface]:
+        interfaces = self.netbox.all_interfaces(device_id=device_ids)
+        extended_ifaces = {
+            interface.id: extend_interface(interface)
+            for interface in interfaces.results
+        }
+
+        ips = self.netbox.all_ip_addresses(interface_id=list(extended_ifaces))
+        for ip in ips.results:
+            extended_ip = extend_ip(ip)
+            interface = extended_ifaces[extended_ip.assigned_object_id]
+            interface.ip_addresses.append(extended_ip)
+        return list(extended_ifaces.values())
+
+    def get_device(
+            self, obj_id, preload_neighbors=False, use_mesh=None,
+            **kwargs,
+    ) -> models.NetboxDevice:
+        device = self.netbox.get_device(obj_id)
+        res = extend_device(device=device)
+        res.interfaces = self._load_interfaces([device.id])
+        return res
+
+    def flush_perf(self):
+        pass
+
+
+def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
+    for subquery in query.globs:
+        if subquery.strip() in device_data.name:
+            return True
+    return False