annet 0.13.4__tar.gz → 3.22.0__tar.gz

{annet-0.13.4/annet.egg-info → annet-3.22.0}/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: annet
-Version: 0.13.4
+Version: 3.22.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 License-File: LICENSE
 License-File: AUTHORS
 Requires-Dist: colorama>=0.4.6
@@ -14,12 +14,20 @@ Requires-Dist: jsonpointer>=2.4
 Requires-Dist: PyYAML>=6.0.1
 Requires-Dist: Pygments>=2.14.0
 Requires-Dist: Mako>=1.2.4
-Requires-Dist: Jinja2>=3.1.2
-Requires-Dist: psutil>=5.8.0
 Requires-Dist: packaging>=23.2
 Requires-Dist: contextlog>=1.1
 Requires-Dist: valkit>=0.1.4
-Requires-Dist: aiohttp>=3.8.4
 Requires-Dist: yarl>=1.8.2
-Requires-Dist: adaptix==3.0.0b5
+Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
+Requires-Dist: ordered-set>=4.1.0
+Provides-Extra: netbox
+Requires-Dist: annetbox[sync]>=0.9.0; extra == "netbox"
+Requires-Dist: requests-cache>=1.2.1; extra == "netbox"
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

annet-3.22.0/README.md

@@ -0,0 +1,89 @@
+# Annet - configuration generation and deploying utility for network equipment
+
+[![Telegram](https://img.shields.io/badge/💬-Telegram-blue)](https://t.me/annet_sup)
+[![PyPI version](https://badge.fury.io/py/annet.svg)](https://pypi.python.org/pypi/annet)
+[![Downloads](https://img.shields.io/pypi/dm/annet.svg)](https://pypistats.org/packages/annet)
+[![License](https://img.shields.io/github/license/annetutil/annet)](https://github.com/annetutil/annet/blob/master/LICENSE)
+[![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/annetutil/annet/setup.yml)](https://github.com/annetutil/annet/actions)
+[![Doc](https://img.shields.io/github/actions/workflow/status/annetutil/annet/docs.yaml?label=docs)](https://annetutil.github.io/annet)
+
+
+Annet is a configuration generator that can translate differences between old and new configurations into sequence of commands. This feature is vital for CLI-based devices, such as Huawei, Cisco IOS, Cisco NX-OS, Juniper. Devices configured via separate config files, Linux, FreeBSD and Cumulus are also supported.
+
+It works this way. Annet `gen`erates configuration for a device by running Python code, which usually goes to the Network Source of Truth, like NetBox. Annet then gets the `diff`erence by getting the configuration from the device and comparing it. Finally, Annet translates the difference into a sequence of commands, called a `patch`. After `deploy`ing these commands, the diff will be empty.
+
+Annet has a number of modes (subcommands):
+
+- ```annet gen``` - generates the entire config for the specified devices or specified parts of it
+- ```annet diff``` - first does gen and then builds diff with current config version
+- ```annet patch``` - first does diff and then generates a list of commands to apply diff on the device
+- ```annet deploy``` - first does patch and then deploys it to the device
+
+Usage help can be obtained by calling ```annet -h``` or for a specific command, such as ```annet gen -h```.
+
+<img src="https://github.com/annetutil/annet/blob/main/docs/_static/annet_demo.gif?raw=true" width="800" />
+
+## Configuration
+
+The path to the configuration file is searched in following order:
+- `ANN_CONTEXT_CONFIG_PATH` env.
+- `~/.annet/context.yml`.
+- `annet/configs/context.yml`.
+
+Config example:
+
+```yaml
+generators:
+  default:
+    - my_annet_generators.example
+
+storage:
+  default:
+    adapter: annet.adapters.file.provider
+    params:
+      path: /path/to/file
+
+context:
+  default:
+    generators: default
+    storage: default
+
+selected_context: default
+```
+
+Environment variable `ANN_SELECTED_CONTEXT` can be used to override `selected_context` parameter.
+
+## Installation
+
+Install from PyPI:
+```shell
+pip install annet
+```
+
+Or install from source:
+```shell
+git clone https://github.com/annetutil/annet
+cd annet
+pip install -e .
+```
+
+## Building doc
+1. Install dependencies:
+```shell
+pip install -r requirements-doc.txt
+```
+2. Build
+```shell
+sphinx-build -M html docs docs-build
+```
+3. Open rendered html in browser
+```shell
+open docs-build/html/index.html  # macOS
+xdg-open docs-build/html/index.html  # Linux
+start docs-build/html/index.html  # Windows
+```
+
+## Links
+
+* [Online Documentation](https://annetutil.github.io/annet/)
+* [Tutorial](https://annetutil.github.io/annet/main/usage/tutorial.html)

{annet-0.13.4 → annet-3.22.0}/annet/__init__.py

@@ -7,14 +7,15 @@ from argparse import SUPPRESS, Namespace
 
 import colorama
 import yaml
-from annet.annlib.errors import (  # pylint: disable=wrong-import-position
-    DeployCancelled,
-    ExecError,
-)
 from contextlog import patch_logging, patch_threading
 from valkit.python import valid_logging_level
 
 import annet.argparse
+from annet.vendors import tabparser  # pylint: disable=unused-import
+from annet.annlib.errors import (  # pylint: disable=wrong-import-position
+    DeployCancelled,
+    ExecError,
+)
 
 
 __all__ = ("DeployCancelled", "ExecError")
@@ -49,13 +50,8 @@ def init(options: Namespace):
         colorama.init = lambda *_, **__: None
     colorama.init()
 
-    # Workaround for Python 3.8.0: https://bugs.python.org/issue38529
-    import asyncio.streams
-    if hasattr(asyncio.streams.StreamReaderProtocol, "_on_reader_gc"):
-        asyncio.streams.StreamReaderProtocol._on_reader_gc = lambda *args, **kwargs: None  # pylint: disable=protected-access
-
 
 def assert_python_version():
-    if sys.version_info < (3, 8, 0):
-        sys.stderr.write("Error: you need python 3.8.0 or higher\n")
+    if sys.version_info < (3, 10, 0):
+        sys.stderr.write("Error: you need python 3.10.0 or higher\n")
         sys.exit(1)

annet-3.22.0/annet/adapters/fetchers/stub/fetcher.py

@@ -0,0 +1,25 @@
+from annet.deploy import Fetcher
+from annet.connectors import AdapterWithConfig
+from typing import Dict, List, Any
+from annet.storage import Device
+
+
+class StubFetcher(Fetcher, AdapterWithConfig):
+    @classmethod
+    def with_config(cls, **kwargs: Dict[str, Any]) -> Fetcher:
+        return cls(**kwargs)
+
+    async def fetch_packages(self,
+                             devices: list[Device],
+                             processes: int = 1,
+                             max_slots: int = 0,
+                             ) -> tuple[dict[Device, str], dict[Device, Any]]:
+        raise NotImplementedError()
+
+    async def fetch(self,
+                    devices: list[Device],
+                    files_to_download: dict[str, list[str]] | None = None,
+                    processes: int = 1,
+                    max_slots: int = 0,
+                    ):
+        raise NotImplementedError()

annet-3.22.0/annet/adapters/file/provider.py

@@ -0,0 +1,280 @@
+import dataclasses
+from dataclasses import dataclass, fields
+from typing import Any, Iterable, List, Optional, Sequence
+
+import yaml
+
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.connectors import AdapterWithName
+from annet.hardware import hardware_connector
+from annet.storage import Device as DeviceProtocol
+from annet.storage import Query, Storage, StorageProvider
+from annet.adapters.netbox.common.manufacturer import get_breed
+
+
+@dataclass
+class Interface(DumpableView):
+    name: str
+    description: str
+    enabled: bool = True
+
+    @property
+    def _dump__list_key(self):
+        return self.name
+
+
+@dataclass
+class DeviceStorage:
+    fqdn: str
+
+    vendor: Optional[str] = None
+    hw_model: Optional[str] = None
+    sw_version: Optional[str] = None
+    breed: Optional[str] = None
+
+    hostname: Optional[str] = None
+    serial: Optional[str] = None
+    id: Optional[str] = None
+    interfaces: Optional[list[Interface]] = None
+    storage: Optional[Storage] = None
+
+    hw: HardwareView = dataclasses.field(init=False)
+
+    def __post_init__(self):
+        if not self.id:
+            self.id = self.fqdn
+        if not self.hostname:
+            self.hostname = self.fqdn.split(".")[0]
+
+        if self.hw_model:
+            hw = HardwareView(self.hw_model, self.sw_version)
+            if self.vendor and self.vendor != hw.vendor:
+                raise Exception(f"Vendor {self.vendor} is not vendor from hw model ({hw.vendor})")
+            else:
+                self.vendor = hw.vendor
+        else:
+            hw_provider = hardware_connector.get()
+            hw: HardwareView = hw_provider.vendor_to_hw(self.vendor)
+            if not hw:
+                raise Exception("unknown vendor")
+            self.hw_model = hw.model
+        self.hw = hw
+
+        if not self.breed:
+            if self.hw.model:
+                parts = self.hw.model.split(maxsplit=1)
+                if len(parts) >= 2:
+                    self.breed = get_breed(parts[0], parts[1])
+            if not self.breed:
+                self.breed = self.hw.vendor
+
+        if isinstance(self.interfaces, list):
+            interfaces = []
+            for iface in self.interfaces:
+                try:
+                    interfaces.append(Interface(**iface))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Interface: %s" % (iface, e))
+            self.interfaces = interfaces
+
+    def set_storage(self, storage: Storage):
+        self.storage = storage
+
+
+@dataclass
+class Device(DeviceProtocol, DumpableView):
+    dev: DeviceStorage
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.fqdn == other.fqdn and self.vendor == other.vendor
+
+    def is_pc(self) -> bool:
+        return False
+
+    @property
+    def hostname(self) -> str:
+        return self.dev.hostname
+
+    @property
+    def fqdn(self) -> str:
+        return self.dev.fqdn
+
+    @property
+    def id(self):
+        return self.dev.id
+
+    @property
+    def storage(self) -> Storage:
+        return self.dev.storage
+
+    @property
+    def hw(self) -> HardwareView:
+        return self.dev.hw
+
+    @property
+    def breed(self) -> str:
+        return self.dev.breed
+
+    @property
+    def neighbours_ids(self):
+        return []
+
+    @property
+    def neighbours_fqdns(self) -> list[str]:
+        return []
+
+    def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: Optional[int]) -> Interface:
+        raise NotImplementedError
+
+    def add_svi(self, svi: int) -> Interface:
+        raise NotImplementedError
+
+    def add_subif(self, interface: str, subif: int) -> Interface:
+        raise NotImplementedError
+
+    def find_interface(self, name: str) -> Optional[Interface]:
+        raise NotImplementedError
+
+    def flush_perf(self):
+        pass
+
+
+@dataclass
+class Devices:
+    devices: list[Device]
+
+    def __post_init__(self):
+        if isinstance(self.devices, list):
+            devices = []
+            for dev in self.devices:
+                try:
+                    devices.append(Device(dev=DeviceStorage(**dev)))
+                except Exception as e:
+                    raise Exception("unable to parse %s as Device: %s" % (dev, e))
+            self.devices = devices
+
+
+class Provider(StorageProvider, AdapterWithName):
+    def storage(self):
+        return storage_factory
+
+    def opts(self):
+        return StorageOpts
+
+    def query(self):
+        return Query
+
+    @classmethod
+    def name(cls) -> str:
+        return "file"
+
+
+@dataclass
+class Query(Query):
+    query: List[str]
+
+    @classmethod
+    def new(cls, query: str | Iterable[str], hosts_range: Optional[slice] = None) -> "Query":
+        if hosts_range is not None:
+            raise ValueError("host_range is not supported")
+        return cls(query=list(query))
+
+    @property
+    def globs(self):
+        return self.query
+
+    def is_empty(self) -> bool:
+        return len(self.query) == 0
+
+
+class StorageOpts:
+    def __init__(self, path: str):
+        self.path = path
+
+    @classmethod
+    def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
+        path = conf_params.get("path")
+        if not path:
+            raise Exception("empty path")
+        return cls(path=path)
+
+
+def storage_factory(opts: StorageOpts) -> Storage:
+    return FS(opts)
+
+
+class FS(Storage):
+    def __init__(self, opts: StorageOpts):
+        self.opts = opts
+        self.inventory: Devices = read_inventory(opts.path, self)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def resolve_fdnds_by_query(self, query: Query) -> list[str]:
+        result = filter_query(self.inventory.devices, query)
+        return [dev.fqdn for dev in result]
+
+    def make_devices(
+        self,
+        query: Query | list,
+        preload_neighbors=False,
+        use_mesh=None,
+        preload_extra_fields=False,
+        **kwargs,
+    ) -> list[Device]:
+        if isinstance(query, list):
+            query = Query.new(query)
+        result = filter_query(self.inventory.devices, query)
+        return result
+
+    def get_device(self, obj_id: str, preload_neighbors=False, use_mesh=None, **kwargs) -> Device:
+        result = filter_query(self.inventory.devices, Query.new(obj_id))
+        if not result:
+            raise Exception("not found")
+        return result[0]
+
+    def flush_perf(self):
+        pass
+
+    def resolve_all_fdnds(self) -> list[str]:
+        return [d.fqdn for d in self.inventory.devices]
+
+    def search_connections(self, device: "Device", neighbor: "Device") -> list[tuple["Interface", "Interface"]]:
+        return []
+
+
+def filter_query(devices: list[Device], query: Query) -> list[Device]:
+    result: list[Device] = []
+    for dev in devices:
+        if dev.fqdn in query.query:
+            result.append(dev)
+    return result
+
+
+def read_inventory(path: str, storage: Storage) -> Devices:
+    with open(path, "r") as f:
+        file_data = yaml.load(f, Loader=yaml.SafeLoader)
+    res = dataclass_from_dict(Devices, file_data)
+    for dev in res.devices:
+        dev.dev.set_storage(storage)
+    return res
+
+
+def dataclass_from_dict(klass: type, d: dict[str, Any]):
+    try:
+        fieldtypes = {f.name: f.type for f in fields(klass)}
+    except TypeError:
+        return d
+    return klass(**{f: dataclass_from_dict(fieldtypes[f], d[f]) for f in d})

annet-3.22.0/annet/adapters/netbox/common/adapter.py

@@ -0,0 +1,87 @@
+from abc import abstractmethod, ABC
+from typing import Generic, TypeVar
+
+from annet.annlib.netdev.views.hardware import HardwareView
+from .manufacturer import get_breed, get_hw
+from .models import NetboxDevice, Interface, IpAddress, Prefix, \
+    FHRPGroupAssignment, FHRPGroup
+
+NetboxDeviceT = TypeVar("NetboxDeviceT", bound=NetboxDevice)
+InterfaceT = TypeVar("InterfaceT", bound=Interface)
+IpAddressT = TypeVar("IpAddressT", bound=IpAddress)
+PrefixT = TypeVar("PrefixT", bound=Prefix)
+FHRPGroupT = TypeVar("FHRPGroupT", bound=FHRPGroup)
+FHRPGroupAssignmentT = TypeVar(
+    "FHRPGroupAssignmentT", bound=FHRPGroupAssignment,
+)
+
+
+def get_device_breed(device: NetboxDeviceT) -> str:
+    if device.device_type and device.device_type.manufacturer:
+        return get_breed(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        )
+    return ""
+
+
+def get_device_hw(device: NetboxDeviceT) -> HardwareView:
+    if device.device_type and device.device_type.manufacturer:
+        return get_hw(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+            device.platform.name if device.platform else "",
+        )
+    return HardwareView("", "")
+
+
+class NetboxAdapter(
+    ABC,
+    Generic[
+        NetboxDeviceT,
+        InterfaceT,
+        IpAddressT,
+        PrefixT,
+        FHRPGroupT,
+        FHRPGroupAssignmentT,
+    ],
+):
+    @abstractmethod
+    def list_fqdns(self, query: dict[str, list[str]] | None = None) -> list[str]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_device(self, device_id: int) -> NetboxDeviceT:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces_by_devices(self, device_ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_interfaces(self, ids: list[int]) -> list[InterfaceT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipaddr_by_ifaces(self, iface_ids: list[int]) -> list[IpAddressT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_fhrp_group_assignments(
+            self, iface_ids: list[int],
+    ) -> list[FHRPGroupAssignmentT]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_fhrp_groups(
+            self, ids: list[int],
+    ) -> list[FHRPGroupT]:
+        raise NotImplementedError()

{annet-0.13.4 → annet-3.22.0}/annet/adapters/netbox/common/client.py

@@ -79,9 +79,11 @@ def collect(func: Func, field: str = "", batch_size: int = 100) -> Func:
 
 
 class BaseNetboxClient(RequestsClient):
-    def __init__(self, url: str, token: str):
+    def __init__(self, url: str, token: str, insecure: bool = False):
         url = url.rstrip("/") + "/api/"
         session = Session()
+        if insecure:
+            session.verify = False
         if token:
             session.headers["Authorization"] = f"Token {token}"
         super().__init__(url, session)

annet-3.22.0/annet/adapters/netbox/common/manufacturer.py

@@ -0,0 +1,44 @@
+from logging import getLogger
+
+from annet.annlib.netdev.views.hardware import HardwareView
+
+logger = getLogger(__name__)
+
+
+def get_hw(manufacturer: str, model: str, platform_name: str):
+    # By some reason Netbox calls Mellanox SN as MSN, so we fix them here
+    if manufacturer == "Mellanox" and model.startswith("MSN"):
+        model = model.replace("MSN", "SN", 1)
+
+    return HardwareView(manufacturer + " " + model, platform_name)
+
+
+def get_breed(manufacturer: str, model: str):
+    hw = get_hw(manufacturer, model, "")
+    if hw.Huawei.CE:
+        return "vrp85"
+    elif hw.Huawei.NE:
+        return "vrp85"
+    elif hw.Huawei:
+        return "vrp55"
+    elif hw.H3C:
+        return "h3c"
+    elif hw.PC.NVIDIA or hw.PC.Mellanox:
+        return "cuml2"
+    elif hw.Juniper:
+        return "jun10"
+    elif hw.Cisco.Nexus:
+        return "nxos"
+    elif hw.Cisco:
+        return "ios12"
+    elif hw.Arista:
+        return "eos4"
+    elif hw.B4com:
+        return "bcom-os"
+    elif hw.RouterOS:
+        return "routeros"
+    elif hw.PC.Moxa or hw.PC.Nebius:
+        return "moxa"
+    elif hw.PC:
+        return "pc"
+    return ""