annet 0.16.8__tar.gz → 0.16.10__tar.gz

{annet-0.16.8/annet.egg-info → annet-0.16.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: annet
-Version: 0.16.8
+Version: 0.16.10
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -23,3 +23,5 @@ Requires-Dist: aiohttp>=3.8.4
 Requires-Dist: yarl>=1.8.2
 Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
+Provides-Extra: netbox
+Requires-Dist: annetbox[sync]>=0.1.8; extra == "netbox"

annet-0.16.10/README.md

@@ -0,0 +1,62 @@
+# Annet - configuration generation and deploying utility for network equipment
+
+Annet is a configuration generator that can translate differences between old and new configurations into sequnce of commands. This feature is vital for CLI-based devices, such as Huawei, Cisco IOS, Cisco NX-OS, Juniper. Devices configured via separate config files, Linux, FreeBSD and Cumulus are also supported.
+
+It works this way. Annet `gen`erates configuration for a device by running Python code, which usually goes to the Network Source of Truth, like NetBox. Annet then gets the `diff`erence by getting the configuration from the device and comparing it. Finally, Annet translates the difference into a sequence of commands, called a `patch`. After `deploy`ing these commands, the diff will be empty.
+
+Annet has a number of modes (subcommands):
+
+- ```annet gen``` - generates the entire config for the specified devices or specified parts of it
+- ```annet diff``` - first does gen and then builds diff with current config version
+- ```annet patch``` - first does diff and then generates a list of commands to apply diff on the device
+- ```annet deploy``` - first does patch and then deploys it to the device
+
+Usage help can be obtained by calling ```annet -h``` or for a specific command, such as ```annet gen -h```.
+
+<img src="https://github.com/annetutil/annet/blob/main/docs/_static/annet_demo.gif?raw=true" width="800" />
+
+## Configuration
+
+The path to the configuration file is searched in following order:
+- `ANN_CONTEXT_CONFIG_PATH` env.
+- `~/.annet/context.yml`.
+- `annet/configs/context.yml`.
+
+Config example:
+
+```yaml
+generators:
+  default:
+    - my_annet_generators.example
+
+storage:
+  default:
+    adapter: annet.adapters.file.provider
+    params:
+      path: /path/to/file
+
+context:
+  default:
+    generators: default
+    storage: default
+
+selected_context: default
+```
+
+Environment variable `ANN_SELECTED_CONTEXT` can be used to override `selected_context` parameter.
+
+## Building doc
+
+1. Install dependencies:
+
+```shell
+pip install -r requirements-doc.txt
+```
+
+2. Build
+
+```shell
+sphinx-build -M html docs docs-build
+```
+
+3. Open rendered html in browser [docs-build/html/index.html](docs-build/html/index.html)

annet-0.16.10/annet/adapters/fetchers/stub/fetcher.py

@@ -0,0 +1,19 @@
+from annet.deploy import Fetcher
+from annet.connectors import AdapterWithConfig
+from typing import Dict, List, Any
+from annet.storage import Device
+
+
+class StubFetcher(Fetcher, AdapterWithConfig):
+    @classmethod
+    def with_config(cls, **kwargs: Dict[str, Any]) -> Fetcher:
+        return cls(**kwargs)
+
+    def fetch_packages(self, devices: List[Device],
+                       processes: int = 1, max_slots: int = 0):
+        raise NotImplementedError()
+
+    def fetch(self, devices: List[Device],
+              files_to_download: Dict[str, List[str]] = None,
+              processes: int = 1, max_slots: int = 0):
+        raise NotImplementedError()

annet-0.16.10/annet/adapters/file/provider.py

@@ -0,0 +1,226 @@
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.storage import Query
+from dataclasses import dataclass, fields
+from typing import List, Iterable, Optional, Any
+from annet.storage import StorageProvider, Storage
+from annet.connectors import AdapterWithName
+from annet.storage import Device as DeviceCls
+from annet.annlib.netdev.views.hardware import vendor_to_hw, HardwareView
+import yaml
+
+
+@dataclass
+class Interface(DumpableView):
+    name: str
+    description: str
+    enabled: bool = True
+
+    @property
+    def _dump__list_key(self):
+        return self.name
+
+
+@dataclass
+class DeviceStorage:
+    fqdn: str
+    vendor: str
+    hostname: Optional[str] = None
+    serial: Optional[str] = None
+    id: Optional[str] = None
+    interfaces: Optional[list[Interface]] = None
+    storage: Optional[Storage] = None
+
+    def __post_init__(self):
+        if not self.id:
+            self.id = self.fqdn
+        if not self.hostname:
+            self.hostname = self.fqdn.split(".")[0]
+        hw = vendor_to_hw(self.vendor)
+        if not hw:
+            raise Exception("unknown vendor")
+        self.hw = hw
+        if isinstance(self.interfaces, list):
+            interfaces = []
+            for iface in self.interfaces:
+                try:
+                    interfaces.append(Interface(**iface))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Interface %s" % (iface, e))
+            self.interfaces = interfaces
+
+    def set_storage(self, storage: Storage):
+        self.storage = storage
+
+
+@dataclass
+class Device(DeviceCls, DumpableView):
+    dev: DeviceStorage
+
+    @property
+    def hostname(self) -> str:
+        return self.dev.hostname
+
+    @property
+    def fqdn(self) -> str:
+        return self.dev.fqdn
+
+    @property
+    def id(self):
+        return self.dev.id
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.fqdn == other.fqdn and self.vendor == other.vendor
+
+    def is_pc(self) -> bool:
+        return False
+
+    @property
+    def storage(self) -> Storage:
+        return self
+
+    @property
+    def hw(self) -> HardwareView:
+        return self.dev.hw
+
+    @property
+    def breed(self) -> str:
+        return self.dev.hw.vendor
+
+    @property
+    def neighbours_ids(self):
+        pass
+
+
+@dataclass
+class Devices:
+    devices: list[Device]
+
+    def __post_init__(self):
+        if isinstance(self.devices, list):
+            devices = []
+            for dev in self.devices:
+                try:
+                    devices.append(Device(dev=DeviceStorage(**dev)))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Device %s" % (dev, e))
+            self.devices = devices
+
+
+class Provider(StorageProvider, AdapterWithName):
+    def storage(self):
+        return storage_factory
+
+    def opts(self):
+        return StorageOpts
+
+    def query(self):
+        return Query
+
+    @classmethod
+    def name(cls) -> str:
+        return "file"
+
+
+@dataclass
+class Query(Query):
+    query: List[str]
+
+    @classmethod
+    def new(cls, query: str | Iterable[str], hosts_range: Optional[slice] = None) -> "Query":
+        if hosts_range is not None:
+            raise ValueError("host_range is not supported")
+        return cls(query=list(query))
+
+    @property
+    def globs(self):
+        return self.query
+
+    def is_empty(self) -> bool:
+        return len(self.query) == 0
+
+
+class StorageOpts:
+    def __init__(self, path: str):
+        self.path = path
+
+    @classmethod
+    def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
+        path = conf_params.get("path")
+        if not path:
+            raise Exception("empty path")
+        return cls(path=path)
+
+
+def storage_factory(opts: StorageOpts) -> Storage:
+    return FS(opts)
+
+
+class FS(Storage):
+    def __init__(self, opts: StorageOpts):
+        self.opts = opts
+        self.inventory: Devices = read_inventory(opts.path, self)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def resolve_fdnds_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def make_devices(
+            self,
+            query: Query | list,
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> list[Device]:
+        if isinstance(query, list):
+            query = Query.new(query)
+        result = filter_query(self.inventory.devices, query)
+        return result
+
+    def get_device(self, obj_id: str, preload_neighbors=False, use_mesh=None, **kwargs) -> Device:
+        result = filter_query(self.inventory.devices, Query.new(obj_id))
+        if not result:
+            raise Exception("not found")
+        return result[0]
+
+    def flush_perf(self):
+        pass
+
+
+def filter_query(devices: list[Device], query: Query) -> list[Device]:
+    result: list[Device] = []
+    for dev in devices:
+        if dev.fqdn in query.query:
+            result.append(dev)
+    return result
+
+
+def read_inventory(path: str, storage: Storage) -> Devices:
+    with open(path, "r") as f:
+        data = f.read()
+    file_data = yaml.load(data, Loader=yaml.BaseLoader)
+    res = dataclass_from_dict(Devices, file_data)
+    for dev in res.devices:
+        dev.dev.set_storage(storage)
+    return res
+
+
+def dataclass_from_dict(klass: type, d: dict[str, Any]):
+    try:
+        fieldtypes = {f.name: f.type for f in fields(klass)}
+    except TypeError:
+        return d
+    return klass(**{f: dataclass_from_dict(fieldtypes[f], d[f]) for f in d})

{annet-0.16.8 → annet-0.16.10}/annet/adapters/netbox/common/manufacturer.py

@@ -51,4 +51,6 @@ def get_breed(manufacturer: str, model: str):
         return "bcom-os"
     elif manufacturer == "MikroTik":
         return "routeros"
+    elif manufacturer == "PC":
+        return "pc"
     return ""

annet-0.16.10/annet/adapters/netbox/common/models.py

@@ -0,0 +1,276 @@
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from ipaddress import ip_interface, IPv6Interface
+from typing import List, Optional, Any, Dict, Sequence, Callable
+
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
+from annet.storage import Storage
+
+
+@dataclass
+class Entity(DumpableView):
+    id: int
+    name: str
+
+    @property
+    def _dump__list_key(self):
+        return self.name
+
+
+@dataclass
+class Label:
+    value: str
+    label: str
+
+
+@dataclass
+class IpFamily:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceType:
+    id: int
+    manufacturer: Entity
+    model: str
+
+
+@dataclass
+class DeviceIp(DumpableView):
+    id: int
+    display: str
+    address: str
+    family: int
+
+    @property
+    def _dump__list_key(self):
+        return self.address
+
+
+@dataclass
+class Prefix(DumpableView):
+    id: int
+    prefix: str
+    site: Optional[Entity]
+    vrf: Optional[Entity]
+    tenant: Optional[Entity]
+    vlan: Optional[Entity]
+    role: Optional[Entity]
+    status: Label
+    is_pool: bool
+    custom_fields: dict[str, Any]
+    created: datetime
+    last_updated: datetime
+    description: Optional[str] = ""
+
+    @property
+    def _dump__list_key(self):
+        return self.prefix
+
+
+@dataclass
+class IpAddress(DumpableView):
+    id: int
+    assigned_object_id: int
+    display: str
+    family: IpFamily
+    address: str
+    status: Label
+    tags: List[Entity]
+    created: datetime
+    last_updated: datetime
+    prefix: Optional[Prefix] = None
+    vrf: Optional[Entity] = None
+
+    @property
+    def _dump__list_key(self):
+        return self.address
+
+
+@dataclass
+class InterfaceConnectedEndpoint(Entity):
+    device: Entity
+
+
+@dataclass
+class InterfaceType:
+    value: str
+    label: str
+
+
+@dataclass
+class InterfaceMode:
+    value: str
+    label: str
+
+
+@dataclass
+class InterfaceVlan(Entity):
+    vid: int
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+    description: str
+    type: InterfaceType
+    connected_endpoints: Optional[list[InterfaceConnectedEndpoint]]
+    mode: Optional[InterfaceMode]
+    untagged_vlan: Optional[InterfaceVlan]
+    tagged_vlans: Optional[List[InterfaceVlan]]
+    display: str = ""
+    ip_addresses: List[IpAddress] = field(default_factory=list)
+    vrf: Optional[Entity] = None
+    mtu: int | None = None
+    lag: Entity | None = None
+    lag_min_links: int | None = None
+
+    def add_addr(self, address_mask: str, vrf: str | None) -> None:
+        addr = ip_interface(address_mask)
+        if vrf is None:
+            vrf_obj = None
+        else:
+            vrf_obj = Entity(id=0, name=vrf)
+
+        if isinstance(addr, IPv6Interface):
+            family = IpFamily(value=6, label="IPv6")
+        else:
+            family = IpFamily(value=4, label="IPv4")
+        self.ip_addresses.append(IpAddress(
+            id=0,
+            display=address_mask,
+            address=address_mask,
+            vrf=vrf_obj,
+            prefix=None,
+            family=family,
+            created=datetime.now(timezone.utc),
+            last_updated=datetime.now(timezone.utc),
+            tags=[],
+            status=Label(value="active", label="Active"),
+            assigned_object_id=self.id,
+        ))
+
+
+@dataclass
+class NetboxDevice(Entity):
+    url: str
+    storage: Storage
+
+    display: str
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: Dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+    fqdn: str
+    hostname: str
+    hw: Optional[HardwareView]
+    breed: str
+
+    interfaces: List[Interface]
+    neighbours: Optional[List["NetboxDevice"]]
+
+    # compat
+    @property
+    def neighbours_fqdns(self) -> list[str]:
+        if not self.neighbours:
+            return []
+        return [dev.fqdn for dev in self.neighbours]
+
+    @property
+    def neighbours_ids(self):
+        if not self.neighbours:
+            return []
+        return [dev.id for dev in self.neighbours]
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.url == other.url
+
+    def is_pc(self) -> bool:
+        return self.device_type.manufacturer.name == "Mellanox" or self.breed == "pc"
+
+    def _make_interface(self, name: str, type: InterfaceType) -> Interface:
+        return Interface(
+            name=name,
+            device=self,
+            enabled=True,
+            description="",
+            type=type,
+            id=0,
+            vrf=None,
+            display=name,
+            untagged_vlan=None,
+            tagged_vlans=[],
+            ip_addresses=[],
+            connected_endpoints=[],
+            mode=None,
+        )
+
+    def _lag_name(self, lag: int) -> str:
+        return lag_name(self.hw, lag)
+
+    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: int | None) -> Interface:
+        new_name = self._lag_name(lag)
+        for target_interface in self.interfaces:
+            if target_interface.name == new_name:
+                return target_interface
+        lag_interface = self._make_interface(
+            name=new_name,
+            type=InterfaceType(value="lag", label="Link Aggregation Group (LAG)"),
+        )
+        lag_interface.lag_min_links = lag_min_links
+        for interface in self.interfaces:
+            if interface.name in ports:
+                interface.lag = lag_interface
+        self.interfaces.append(lag_interface)
+        return lag_interface
+
+    def _svi_name(self, svi: int) -> str:
+        return svi_name(self.hw, svi)
+
+    def add_svi(self, svi: int) -> Interface:
+        name = self._svi_name(svi)
+        for interface in self.interfaces:
+            if interface.name == name:
+                return interface
+        interface = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(interface)
+        return interface
+
+    def _subif_name(self, interface: str, subif: int) -> str:
+        return f"{interface}.{subif}"
+
+    def add_subif(self, interface: str, subif: int) -> Interface:
+        name = self._subif_name(interface, subif)
+        for target_port in self.interfaces:
+            if target_port.name == name:
+                return target_port
+        target_port = self._make_interface(
+            name=name,
+            type=InterfaceType("virtual", "Virtual")
+        )
+        self.interfaces.append(target_port)
+        return target_port

{annet-0.16.8 → annet-0.16.10}/annet/adapters/netbox/v37/storage.py

@@ -15,8 +15,7 @@ from annet.adapters.netbox.common.manufacturer import (
 from annet.adapters.netbox.common.query import NetboxQuery
 from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
 from annet.annlib.netdev.views.hardware import HardwareView
-from annet.storage import Storage
-
+from annet.storage import Storage, Device, Interface
 
 logger = getLogger(__name__)
 
@@ -68,7 +67,9 @@ def extend_device(
     return res
 
 
-@impl_converter
+@impl_converter(
+    recipe=[link_constant(P[models.Interface].lag_min_links, value=None)],
+)
 def extend_interface(
         interface: api_models.Interface,
         ip_addresses: List[models.IpAddress],
@@ -89,6 +90,7 @@ class NetboxStorageV37(Storage):
             url=opts.url,
             token=opts.token,
         )
+        self._all_fqdns: Optional[list[str]] = None
 
     def __enter__(self):
         return self
@@ -106,6 +108,14 @@ class NetboxStorageV37(Storage):
             d.name for d in self._load_devices(query)
         ]
 
+    def resolve_all_fdnds(self) -> list[str]:
+        if self._all_fqdns is None:
+            self._all_fqdns = [
+                d.name
+                for d in self.netbox.dcim_all_devices().results
+            ]
+        return self._all_fqdns
+
     def make_devices(
             self,
             query: Union[NetboxQuery, list],
@@ -218,6 +228,24 @@ class NetboxStorageV37(Storage):
     def flush_perf(self):
         pass
 
+    def search_connections(self, device: Device, neighbor: Device) -> list[tuple[Interface, Interface]]:
+        if device.storage is not self:
+            raise ValueError("device does not belong to this storage")
+        if neighbor.storage is not self:
+            raise ValueError("neighbor does not belong to this storage")
+        # both devices are NetboxDevice if they are loaded from this storage
+        res = []
+        for local_port in device.interfaces:
+            if not local_port.connected_endpoints:
+                continue
+            for endpoint in local_port.connected_endpoints:
+                if endpoint.device.id == neighbor.id:
+                    for remote_port in neighbor.interfaces:
+                        if remote_port.name == endpoint.name:
+                            res.append((local_port, remote_port))
+                            break
+        return res
+
 
 def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
     for subquery in query.globs:

{annet-0.16.8 → annet-0.16.10}/annet/annlib/netdev/views/hardware.py

@@ -123,3 +123,34 @@ def vendor_to_hw(vendor):
         None,
     )
     return hw
+
+
+def lag_name(hw: HardwareView, nlagg: int) -> str:
+    if hw.Huawei:
+        return f"Eth-Trunk{nlagg}"
+    if hw.Cisco:
+        return f"port-channel{nlagg}"
+    if hw.Nexus:
+        return f"port-channel{nlagg}"
+    if hw.Arista:
+        return f"Port-Channel{nlagg}"
+    if hw.Juniper:
+        return f"ae{nlagg}"
+    if hw.Nokia:
+        return f"lag-{nlagg}"
+    if hw.PC.Whitebox:
+        return f"bond{nlagg}"
+    if hw.PC:
+        return f"lagg{nlagg}"
+    if hw.Nokia:
+        return f"lagg-{nlagg}"
+    raise NotImplementedError(hw)
+
+
+def svi_name(hw: HardwareView, num: int) -> str:
+    if hw.Juniper:
+        return f"irb.{num}"
+    elif hw.Huawei:
+        return f"Vlanif{num}"
+    else:
+        return f"vlan{num}"