annet 0.16.8__py3-none-any.whl → 0.16.9__py3-none-any.whl

annet/adapters/fetchers/stub/fetcher.py

@@ -0,0 +1,19 @@
+from annet.deploy import Fetcher
+from annet.connectors import AdapterWithConfig
+from typing import Dict, List, Any
+from annet.storage import Device
+
+
+class StubFetcher(Fetcher, AdapterWithConfig):
+    @classmethod
+    def with_config(cls, **kwargs: Dict[str, Any]) -> Fetcher:
+        return cls(**kwargs)
+
+    def fetch_packages(self, devices: List[Device],
+                       processes: int = 1, max_slots: int = 0):
+        raise NotImplementedError()
+
+    def fetch(self, devices: List[Device],
+              files_to_download: Dict[str, List[str]] = None,
+              processes: int = 1, max_slots: int = 0):
+        raise NotImplementedError()

annet/adapters/file/provider.py

@@ -0,0 +1,226 @@
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.storage import Query
+from dataclasses import dataclass, fields
+from typing import List, Iterable, Optional, Any
+from annet.storage import StorageProvider, Storage
+from annet.connectors import AdapterWithName
+from annet.storage import Device as DeviceCls
+from annet.annlib.netdev.views.hardware import vendor_to_hw, HardwareView
+import yaml
+
+
+@dataclass
+class Interface(DumpableView):
+    name: str
+    description: str
+    enabled: bool = True
+
+    @property
+    def _dump__list_key(self):
+        return self.name
+
+
+@dataclass
+class DeviceStorage:
+    fqdn: str
+    vendor: str
+    hostname: Optional[str] = None
+    serial: Optional[str] = None
+    id: Optional[str] = None
+    interfaces: Optional[list[Interface]] = None
+    storage: Optional[Storage] = None
+
+    def __post_init__(self):
+        if not self.id:
+            self.id = self.fqdn
+        if not self.hostname:
+            self.hostname = self.fqdn.split(".")[0]
+        hw = vendor_to_hw(self.vendor)
+        if not hw:
+            raise Exception("unknown vendor")
+        self.hw = hw
+        if isinstance(self.interfaces, list):
+            interfaces = []
+            for iface in self.interfaces:
+                try:
+                    interfaces.append(Interface(**iface))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Interface %s" % (iface, e))
+            self.interfaces = interfaces
+
+    def set_storage(self, storage: Storage):
+        self.storage = storage
+
+
+@dataclass
+class Device(DeviceCls, DumpableView):
+    dev: DeviceStorage
+
+    @property
+    def hostname(self) -> str:
+        return self.dev.hostname
+
+    @property
+    def fqdn(self) -> str:
+        return self.dev.fqdn
+
+    @property
+    def id(self):
+        return self.dev.id
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.fqdn == other.fqdn and self.vendor == other.vendor
+
+    def is_pc(self) -> bool:
+        return False
+
+    @property
+    def storage(self) -> Storage:
+        return self
+
+    @property
+    def hw(self) -> HardwareView:
+        return self.dev.hw
+
+    @property
+    def breed(self) -> str:
+        return self.dev.hw.vendor
+
+    @property
+    def neighbours_ids(self):
+        pass
+
+
+@dataclass
+class Devices:
+    devices: list[Device]
+
+    def __post_init__(self):
+        if isinstance(self.devices, list):
+            devices = []
+            for dev in self.devices:
+                try:
+                    devices.append(Device(dev=DeviceStorage(**dev)))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Device %s" % (dev, e))
+            self.devices = devices
+
+
+class Provider(StorageProvider, AdapterWithName):
+    def storage(self):
+        return storage_factory
+
+    def opts(self):
+        return StorageOpts
+
+    def query(self):
+        return Query
+
+    @classmethod
+    def name(cls) -> str:
+        return "file"
+
+
+@dataclass
+class Query(Query):
+    query: List[str]
+
+    @classmethod
+    def new(cls, query: str | Iterable[str], hosts_range: Optional[slice] = None) -> "Query":
+        if hosts_range is not None:
+            raise ValueError("host_range is not supported")
+        return cls(query=list(query))
+
+    @property
+    def globs(self):
+        return self.query
+
+    def is_empty(self) -> bool:
+        return len(self.query) == 0
+
+
+class StorageOpts:
+    def __init__(self, path: str):
+        self.path = path
+
+    @classmethod
+    def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
+        path = conf_params.get("path")
+        if not path:
+            raise Exception("empty path")
+        return cls(path=path)
+
+
+def storage_factory(opts: StorageOpts) -> Storage:
+    return FS(opts)
+
+
+class FS(Storage):
+    def __init__(self, opts: StorageOpts):
+        self.opts = opts
+        self.inventory: Devices = read_inventory(opts.path, self)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def resolve_fdnds_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def make_devices(
+            self,
+            query: Query | list,
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> list[Device]:
+        if isinstance(query, list):
+            query = Query.new(query)
+        result = filter_query(self.inventory.devices, query)
+        return result
+
+    def get_device(self, obj_id: str, preload_neighbors=False, use_mesh=None, **kwargs) -> Device:
+        result = filter_query(self.inventory.devices, Query.new(obj_id))
+        if not result:
+            raise Exception("not found")
+        return result[0]
+
+    def flush_perf(self):
+        pass
+
+
+def filter_query(devices: list[Device], query: Query) -> list[Device]:
+    result: list[Device] = []
+    for dev in devices:
+        if dev.fqdn in query.query:
+            result.append(dev)
+    return result
+
+
+def read_inventory(path: str, storage: Storage) -> Devices:
+    with open(path, "r") as f:
+        data = f.read()
+    file_data = yaml.load(data, Loader=yaml.BaseLoader)
+    res = dataclass_from_dict(Devices, file_data)
+    for dev in res.devices:
+        dev.dev.set_storage(storage)
+    return res
+
+
+def dataclass_from_dict(klass: type, d: dict[str, Any]):
+    try:
+        fieldtypes = {f.name: f.type for f in fields(klass)}
+    except TypeError:
+        return d
+    return klass(**{f: dataclass_from_dict(fieldtypes[f], d[f]) for f in d})

annet/adapters/netbox/common/models.py

@@ -1,9 +1,10 @@
 from dataclasses import dataclass, field
-from datetime import datetime
-from typing import List, Optional, Any, Dict
+from datetime import datetime, timezone
+from ipaddress import ip_interface, IPv6Interface
+from typing import List, Optional, Any, Dict, Sequence, Callable
 
 from annet.annlib.netdev.views.dump import DumpableView
-from annet.annlib.netdev.views.hardware import HardwareView
+from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
 from annet.storage import Storage
 
 
@@ -124,6 +125,33 @@ class Interface(Entity):
     ip_addresses: List[IpAddress] = field(default_factory=list)
     vrf: Optional[Entity] = None
     mtu: int | None = None
+    lag: Entity | None = None
+    lag_min_links: int | None = None
+
+    def add_addr(self, address_mask: str, vrf: str | None) -> None:
+        addr = ip_interface(address_mask)
+        if vrf is None:
+            vrf_obj = None
+        else:
+            vrf_obj = Entity(id=0, name=vrf)
+
+        if isinstance(addr, IPv6Interface):
+            family = IpFamily(value=6, label="IPv6")
+        else:
+            family = IpFamily(value=4, label="IPv4")
+        self.ip_addresses.append(IpAddress(
+            id=0,
+            display=address_mask,
+            address=address_mask,
+            vrf=vrf_obj,
+            prefix=None,
+            family=family,
+            created=datetime.now(timezone.utc),
+            last_updated=datetime.now(timezone.utc),
+            tags=[],
+            status=Label(value="active", label="Active"),
+            assigned_object_id=self.id,
+        ))
 
 
 @dataclass
@@ -160,6 +188,17 @@ class NetboxDevice(Entity):
     neighbours: Optional[List["NetboxDevice"]]
 
     # compat
+    @property
+    def neighbours_fqdns(self) -> list[str]:
+        if not self.neighbours:
+            return []
+        return [dev.fqdn for dev in self.neighbours]
+
+    @property
+    def neighbours_ids(self):
+        if not self.neighbours:
+            return []
+        return [dev.id for dev in self.neighbours]
 
     def __hash__(self):
         return hash((self.id, type(self)))
@@ -169,3 +208,69 @@ class NetboxDevice(Entity):
 
     def is_pc(self) -> bool:
         return self.device_type.manufacturer.name == "Mellanox"
+
+    def _make_interface(self, name: str, type: InterfaceType) -> Interface:
+        return Interface(
+            name=name,
+            device=self,
+            enabled=True,
+            description="",
+            type=type,
+            id=0,
+            vrf=None,
+            display=name,
+            untagged_vlan=None,
+            tagged_vlans=[],
+            ip_addresses=[],
+            connected_endpoints=[],
+            mode=None,
+        )
+
+    def _lag_name(self, lag: int) -> str:
+        return lag_name(self.hw, lag)
+
+    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: int | None) -> Interface:
+        new_name = self._lag_name(lag)
+        for target_interface in self.interfaces:
+            if target_interface.name == new_name:
+                return target_interface
+        lag_interface = self._make_interface(
+            name=new_name,
+            type=InterfaceType(value="lag", label="Link Aggregation Group (LAG)"),
+        )
+        lag_interface.lag_min_links = lag_min_links
+        for interface in self.interfaces:
+            if interface.name in ports:
+                interface.lag = lag_interface
+        self.interfaces.append(lag_interface)
+        return lag_interface
+
+    def _svi_name(self, svi: int) -> str:
+        return svi_name(self.hw, svi)
+
+    def add_svi(self, svi: int) -> Interface:
+        name = self._svi_name(svi)
+        for interface in self.interfaces:
+            if interface.name == name:
+                return interface
+        interface = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(interface)
+        return interface
+
+    def _subif_name(self, interface: str, subif: int) -> str:
+        return f"{interface}.{subif}"
+
+    def add_subif(self, interface: str, subif: int) -> Interface:
+        name = self._subif_name(interface, subif)
+        for target_port in self.interfaces:
+            if target_port.name == name:
+                return target_port
+        target_port = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(target_port)
+        return target_port

annet/adapters/netbox/v37/storage.py

@@ -15,8 +15,7 @@ from annet.adapters.netbox.common.manufacturer import (
 from annet.adapters.netbox.common.query import NetboxQuery
 from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
 from annet.annlib.netdev.views.hardware import HardwareView
-from annet.storage import Storage
-
+from annet.storage import Storage, Device, Interface
 
 logger = getLogger(__name__)
 
@@ -68,7 +67,9 @@ def extend_device(
     return res
 
 
-@impl_converter
+@impl_converter(
+    recipe=[link_constant(P[models.Interface].lag_min_links, value=None)],
+)
 def extend_interface(
         interface: api_models.Interface,
         ip_addresses: List[models.IpAddress],
@@ -89,6 +90,7 @@ class NetboxStorageV37(Storage):
             url=opts.url,
             token=opts.token,
         )
+        self._all_fqdns: Optional[list[str]] = None
 
     def __enter__(self):
         return self
@@ -106,6 +108,14 @@ class NetboxStorageV37(Storage):
             d.name for d in self._load_devices(query)
         ]
 
+    def resolve_all_fdnds(self) -> list[str]:
+        if self._all_fqdns is None:
+            self._all_fqdns = [
+                d.name
+                for d in self.netbox.dcim_all_devices().results
+            ]
+        return self._all_fqdns
+
     def make_devices(
             self,
             query: Union[NetboxQuery, list],
@@ -218,6 +228,24 @@ class NetboxStorageV37(Storage):
     def flush_perf(self):
         pass
 
+    def search_connections(self, device: Device, neighbor: Device) -> list[tuple[Interface, Interface]]:
+        if device.storage is not self:
+            raise ValueError("device does not belong to this storage")
+        if neighbor.storage is not self:
+            raise ValueError("neighbor does not belong to this storage")
+        # both devices are NetboxDevice if they are loaded from this storage
+        res = []
+        for local_port in device.interfaces:
+            if not local_port.connected_endpoints:
+                continue
+            for endpoint in local_port.connected_endpoints:
+                if endpoint.device.id == neighbor.id:
+                    for remote_port in neighbor.interfaces:
+                        if remote_port.name == endpoint.name:
+                            res.append((local_port, remote_port))
+                            break
+        return res
+
 
 def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
     for subquery in query.globs:

annet/annlib/netdev/views/hardware.py

@@ -123,3 +123,34 @@ def vendor_to_hw(vendor):
         None,
     )
     return hw
+
+
+def lag_name(hw: HardwareView, nlagg: int) -> str:
+    if hw.Huawei:
+        return f"Eth-Trunk{nlagg}"
+    if hw.Cisco:
+        return f"port-channel{nlagg}"
+    if hw.Nexus:
+        return f"port-channel{nlagg}"
+    if hw.Arista:
+        return f"Port-Channel{nlagg}"
+    if hw.Juniper:
+        return f"ae{nlagg}"
+    if hw.Nokia:
+        return f"lag-{nlagg}"
+    if hw.PC.Whitebox:
+        return f"bond{nlagg}"
+    if hw.PC:
+        return f"lagg{nlagg}"
+    if hw.Nokia:
+        return f"lagg-{nlagg}"
+    raise NotImplementedError(hw)
+
+
+def svi_name(hw: HardwareView, num: int) -> str:
+    if hw.Juniper:
+        return f"irb.{num}"
+    elif hw.Huawei:
+        return f"Vlanif{num}"
+    else:
+        return f"vlan{num}"