annet 2.5.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

annet/adapters/file/provider.py

@@ -1,13 +1,16 @@
-from annet.annlib.netdev.views.dump import DumpableView
-from annet.storage import Query
+import dataclasses
 from dataclasses import dataclass, fields
-from typing import List, Iterable, Optional, Any, Sequence
-from annet.storage import StorageProvider, Storage
-from annet.connectors import AdapterWithName
-from annet.storage import Device as DeviceCls
-from annet.annlib.netdev.views.hardware import vendor_to_hw, HardwareView
+from typing import Any, Iterable, List, Optional, Sequence
+
 import yaml
 
+from annet.annlib.netdev.views.dump import DumpableView
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.connectors import AdapterWithName
+from annet.hardware import hardware_connector
+from annet.storage import Device as DeviceProtocol
+from annet.storage import Query, Storage, StorageProvider
+
 
 @dataclass
 class Interface(DumpableView):
@@ -23,29 +26,46 @@ class Interface(DumpableView):
 @dataclass
 class DeviceStorage:
     fqdn: str
-    vendor: str
+
+    vendor: Optional[str] = None
+    hw_model: Optional[str] = None
+    sw_version: Optional[str] = None
+
     hostname: Optional[str] = None
     serial: Optional[str] = None
     id: Optional[str] = None
     interfaces: Optional[list[Interface]] = None
     storage: Optional[Storage] = None
 
+    hw: HardwareView = dataclasses.field(init=False)
+
     def __post_init__(self):
         if not self.id:
             self.id = self.fqdn
         if not self.hostname:
             self.hostname = self.fqdn.split(".")[0]
-        hw = vendor_to_hw(self.vendor)
-        if not hw:
-            raise Exception("unknown vendor")
+
+        if self.hw_model:
+            hw = HardwareView(self.hw_model, self.sw_version)
+            if self.vendor and self.vendor != hw.vendor:
+                raise Exception(f"Vendor {self.vendor} is not vendor from hw model ({hw.vendor})")
+            else:
+                self.vendor = hw.vendor
+        else:
+            hw_provider = hardware_connector.get()
+            hw: HardwareView = hw_provider.vendor_to_hw(self.vendor)
+            if not hw:
+                raise Exception("unknown vendor")
+            self.hw_model = hw.model
         self.hw = hw
+
         if isinstance(self.interfaces, list):
             interfaces = []
             for iface in self.interfaces:
                 try:
                     interfaces.append(Interface(**iface))
                 except Exception as e:
-                    raise Exception("unable to parse %s as Interface %s" % (iface, e))
+                    raise Exception("unable to parse %s as Interface: %s" % (iface, e))
             self.interfaces = interfaces
 
     def set_storage(self, storage: Storage):
@@ -53,7 +73,7 @@ class DeviceStorage:
 
 
 @dataclass
-class Device(DeviceCls, DumpableView):
+class Device(DeviceProtocol, DumpableView):
     dev: DeviceStorage
 
     def __hash__(self):
@@ -79,7 +99,7 @@ class Device(DeviceCls, DumpableView):
 
     @property
     def storage(self) -> Storage:
-        return self
+        return self.dev.storage
 
     @property
     def hw(self) -> HardwareView:
@@ -91,7 +111,11 @@ class Device(DeviceCls, DumpableView):
 
     @property
     def neighbours_ids(self):
-        pass
+        return []
+
+    @property
+    def neighbours_fqdns(self) -> list[str]:
+        return []
 
     def make_lag(self, lag: int, ports: Sequence[str], lag_min_links: Optional[int]) -> Interface:
         raise NotImplementedError
@@ -105,8 +129,8 @@ class Device(DeviceCls, DumpableView):
     def find_interface(self, name: str) -> Optional[Interface]:
         raise NotImplementedError
 
-    def neighbours_fqdns(self) -> list[str]:
-        return []
+    def flush_perf(self):
+        pass
 
 
 @dataclass
@@ -120,7 +144,7 @@ class Devices:
                 try:
                     devices.append(Device(dev=DeviceStorage(**dev)))
                 except Exception as e:
-                    raise Exception("unable to parse %s as Device %s" % (dev, e))
+                    raise Exception("unable to parse %s as Device: %s" % (dev, e))
             self.devices = devices
 
 
@@ -193,12 +217,12 @@ class FS(Storage):
         return [dev.fqdn for dev in result]
 
     def make_devices(
-            self,
-            query: Query | list,
-            preload_neighbors=False,
-            use_mesh=None,
-            preload_extra_fields=False,
-            **kwargs,
+        self,
+        query: Query | list,
+        preload_neighbors=False,
+        use_mesh=None,
+        preload_extra_fields=False,
+        **kwargs,
     ) -> list[Device]:
         if isinstance(query, list):
             query = Query.new(query)
@@ -231,8 +255,7 @@ def filter_query(devices: list[Device], query: Query) -> list[Device]:
 
 def read_inventory(path: str, storage: Storage) -> Devices:
     with open(path, "r") as f:
-        data = f.read()
-    file_data = yaml.load(data, Loader=yaml.BaseLoader)
+        file_data = yaml.load(f, Loader=yaml.SafeLoader)
     res = dataclass_from_dict(Devices, file_data)
     for dev in res.devices:
         dev.dev.set_storage(storage)

annet/adapters/netbox/common/manufacturer.py

@@ -4,31 +4,13 @@ from annet.annlib.netdev.views.hardware import HardwareView
 
 logger = getLogger(__name__)
 
-_VENDORS = {
-    "cisco": "Cisco",
-    "catalyst": "Cisco Catalyst",
-    "nexus": "Cisco Nexus",
-    "huawei": "Huawei",
-    "optixtrans": "Huawei OptiXtrans",
-    "juniper": "Juniper",
-    "arista": "Arista",
-    "pc": "PC",
-    "nokia": "Nokia",
-    "aruba": "Aruba",
-    "routeros": "RouterOS",
-    "ribbon": "Ribbon",
-    "b4com": "B4com",
-    "h3c": "H3C",
-}
-
 
 def get_hw(manufacturer: str, model: str, platform_name: str):
-    # by some reason Netbox calls Mellanox SN as MSN, so we fix them here
+    # By some reason Netbox calls Mellanox SN as MSN, so we fix them here
     if manufacturer == "Mellanox" and model.startswith("MSN"):
         model = model.replace("MSN", "SN", 1)
-    vendor = manufacturer + " " + model
-    hw = HardwareView(_VENDORS.get(vendor.lower(), vendor), platform_name)
-    return hw
+
+    return HardwareView(manufacturer + " " + model, platform_name)
 
 
 def get_breed(manufacturer: str, model: str):

annet/adapters/netbox/common/models.py

@@ -5,8 +5,9 @@ from ipaddress import ip_interface, IPv6Interface
 from typing import List, Optional, Any, Dict, Sequence, TypeVar, Generic
 
 from annet.annlib.netdev.views.dump import DumpableView
-from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
+from annet.annlib.netdev.views.hardware import HardwareView, lag_name
 from annet.storage import Storage
+from annet.vendors import registry_connector
 
 
 @dataclass
@@ -140,7 +141,7 @@ class Interface(Entity, Generic[_IpAddressT]):
     mode: Optional[InterfaceMode]
     untagged_vlan: Optional[InterfaceVlan]
     tagged_vlans: Optional[List[InterfaceVlan]]
-    tags: List[EntityWithSlug]
+    tags: List[EntityWithSlug] = field(default_factory=list)
     display: str = ""
     ip_addresses: List[_IpAddressT] = field(default_factory=list)
     vrf: Optional[Entity] = None
@@ -259,7 +260,7 @@ class NetboxDevice(Entity, Generic[_InterfaceT]):
         return lag_interface
 
     def _svi_name(self, svi: int) -> str:
-        return svi_name(self.hw, svi)
+        return registry_connector.get().match(self.hw).svi_name(svi)
 
     def add_svi(self, svi: int) -> _InterfaceT:
         name = self._svi_name(svi)

annet/adapters/netbox/v24/models.py

@@ -1,11 +1,12 @@
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from ipaddress import ip_interface, IPv6Interface
-from typing import List, Optional, Any, Dict, Sequence, Callable
+from typing import List, Optional, Any, Dict, Sequence
 
 from annet.annlib.netdev.views.dump import DumpableView
-from annet.annlib.netdev.views.hardware import HardwareView, lag_name, svi_name
+from annet.annlib.netdev.views.hardware import HardwareView, lag_name
 from annet.storage import Storage
+from annet.vendors import registry_connector
 
 
 @dataclass
@@ -259,7 +260,7 @@ class NetboxDevice(Entity):
         return lag_interface
 
     def _svi_name(self, svi: int) -> str:
-        return svi_name(self.hw, svi)
+        return registry_connector.get().match(self.hw).svi_name(svi)
 
     def add_svi(self, svi: int) -> Interface:
         name = self._svi_name(svi)

annet/annet.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import logging
 import sys
 
 import annet
@@ -18,8 +19,14 @@ def main():
     parser.add_commands(parser.find_subcommands(cli.list_subcommands()))
     try:
         return parser.dispatch(pre_call=annet.init, add_help_command=True)
-    except (generators.GeneratorError, annet.ExecError):
+    except (generators.GeneratorError, annet.ExecError) as e:
+        logging.error(e)
         return 1
+    except BaseException as e:
+        if formatted_output := getattr(e, "formatted_output", None):
+            logging.error(formatted_output)
+            return 1
+        raise e
 
 
 if __name__ == "__main__":

annet/annlib/netdev/devdb/data/devdb.json

@@ -6,6 +6,7 @@
     "Cisco.ASR": " ASR",
     "Cisco.ASR.ASR9000": " 9\\d{3}",
     "Cisco.XRV": "XRv",
+    "Cisco.XR": " 8[\\d\\d\\d]",
     "Cisco.Catalyst": " (Catalyst |WS-C)",
     "Cisco.Catalyst.C2900": "29\\d\\d",
     "Cisco.Catalyst.C2900.C2950": "2950",

annet/annlib/netdev/views/hardware.py

@@ -1,3 +1,4 @@
+import functools
 from typing import Optional
 
 from annet.annlib.netdev.devdb import parse_hw_model
@@ -51,20 +52,31 @@ class HardwareLeaf(DumpableView):
 
 
 class HardwareView(HardwareLeaf):
-    def __init__(self, hw_model, sw_version):
-        (true_sequences, false_sequences) = parse_hw_model(hw_model or "")
+    def __init__(self, hw_model, sw_version: Optional[str] = None):
+        true_sequences, false_sequences = parse_hw_model(hw_model or "")
         super().__init__((), true_sequences, false_sequences)
         self.model = hw_model or ""
         self._soft = sw_version or ""
 
     @property
     def vendor(self) -> Optional[str]:
-        return hw_to_vendor(self)
+        from annet.hardware import hardware_connector
+        return hardware_connector.get().hw_to_vendor(self)
 
     @property
     def soft(self) -> str:
         return self._soft
 
+    @soft.setter
+    def soft(self, value: str) -> None:
+        self._soft = value
+
+    def match(self, expr: str) -> bool:
+        dev_path = expr.split(".")
+        if dev_path and dev_path[0] == "hw":
+            dev_path = dev_path[1:]
+        return bool(functools.reduce(getattr, dev_path, self))
+
     def __hash__(self):
         return hash(self.model)
 
@@ -72,59 +84,6 @@ class HardwareView(HardwareLeaf):
         return self.model == other.model
 
 
-def hw_to_vendor(hw: HardwareView) -> Optional[str]:
-    if hw.Nexus:
-        return "nexus"
-    elif hw.Cisco:
-        return "cisco"
-    elif hw.OptiXtrans:
-        return "optixtrans"
-    elif hw.Huawei:
-        return "huawei"
-    elif hw.Juniper:
-        return "juniper"
-    elif hw.Arista:
-        return "arista"
-    elif hw.PC:
-        return "pc"
-    elif hw.Nokia:
-        return "nokia"
-    elif hw.RouterOS:
-        return "routeros"
-    elif hw.Aruba:
-        return "aruba"
-    elif hw.Ribbon:
-        return "ribbon"
-    elif hw.H3C:
-        return "h3c"
-    elif hw.B4com:
-        return "b4com"
-    return None
-
-
-def vendor_to_hw(vendor):
-    hw = HardwareView(
-        {
-            "cisco": "Cisco",
-            "catalyst": "Cisco Catalyst",
-            "nexus": "Cisco Nexus",
-            "huawei": "Huawei",
-            "optixtrans": "Huawei OptiXtrans",
-            "juniper": "Juniper",
-            "arista": "Arista",
-            "pc": "PC",
-            "nokia": "Nokia",
-            "aruba": "Aruba",
-            "routeros": "RouterOS",
-            "ribbon": "Ribbon",
-            "h3c": "H3C",
-            "b4com": "B4com",
-        }.get(vendor.lower(), vendor),
-        None,
-    )
-    return hw
-
-
 def lag_name(hw: HardwareView, nlagg: int) -> str:
     if hw.Huawei:
         return f"Eth-Trunk{nlagg}"
@@ -145,14 +104,3 @@ def lag_name(hw: HardwareView, nlagg: int) -> str:
     if hw.Nokia:
         return f"lagg-{nlagg}"
     raise NotImplementedError(hw)
-
-
-def svi_name(hw: HardwareView, num: int) -> str:
-    if hw.Juniper:
-        return f"irb.{num}"
-    elif hw.Huawei:
-        return f"Vlanif{num}"
-    elif hw.Arista or hw.Cisco:
-        return f"Vlan{num}"
-    else:
-        return f"vlan{num}"

annet/annlib/patching.py

@@ -19,6 +19,7 @@ from .rulebook.common import call_diff_logic
 from .rulebook.common import default as common_default
 from .tabparser import CommonFormatter
 from .types import Diff, Op
+from ..vendors import registry_connector
 
 
 # =====
@@ -224,14 +225,15 @@ class Orderer:
         return (f_order or 0), cmd_direct, odict(children), f_rule
 
     def order_config(self, config):
-        if self.vendor not in platform.VENDOR_REVERSES:
+        if self.vendor not in registry_connector.get():
             return config
-
-        ordered = []
-        reverse_prefix = platform.VENDOR_REVERSES[self.vendor]
         if not config:
             return odict()
-        for (row, children) in config.items():
+
+        ordered = []
+        reverse_prefix = registry_connector.get()[self.vendor].reverse
+
+        for row, children in config.items():
             cmd_direct = not row.startswith(reverse_prefix)
             (order, direct, rb, _) = self.get_order(row, cmd_direct)
             child_orderer = Orderer(rb, self.vendor)

annet/annlib/rbparser/acl.py

@@ -5,7 +5,8 @@ from typing import Any, Callable, List, Optional
 from valkit import add_validator_magic
 from valkit.common import valid_bool, valid_number, valid_string_list
 
-from . import platform, syntax
+from annet.vendors import registry_connector
+from . import syntax
 
 
 # =====
@@ -13,7 +14,7 @@ from . import platform, syntax
 def compile_acl_text(text, vendor, allow_ignore=False):
     return _compile_acl(
         trees=[syntax.parse_text(text, _PARAMS_SCHEME)],
-        reverse_prefix=platform.VENDOR_REVERSES[vendor],
+        reverse_prefix=registry_connector.get()[vendor].reverse,
         allow_ignore=allow_ignore,
         vendor=vendor,
     )

annet/annlib/rbparser/ordering.py

@@ -4,7 +4,8 @@ from collections import OrderedDict as odict
 
 from valkit.common import valid_bool, valid_string_list
 
-from . import platform, syntax
+from annet.vendors import registry_connector
+from . import syntax
 
 
 # =====
@@ -25,7 +26,7 @@ def compile_ordering_text(text, vendor):
                 "default": None,
             }
         }),
-        reverse_prefix=platform.VENDOR_REVERSES[vendor],
+        reverse_prefix=registry_connector.get()[vendor].reverse,
     )
 
 

annet/annlib/rbparser/platform.py

@@ -3,6 +3,7 @@ VENDOR_REVERSES = {
     "h3c": "undo",
     "optixtrans": "undo",
     "cisco": "no",
+    "iosxr": "no",
     "nexus": "no",
     "juniper": "delete",
     "arista": "no",
@@ -18,6 +19,7 @@ VENDOR_DIFF = {
     "huawei": "common.default_diff",
     "optixtrans": "common.default_diff",
     "cisco": "common.default_diff",
+    "iosxr": "common.default_diff",
     "nexus": "common.default_diff",
     "juniper": "juniper.default_diff",
     "arista": "common.default_diff",
@@ -33,6 +35,7 @@ VENDOR_DIFF_ORDERED = {
     "huawei": "common.ordered_diff",
     "optixtrans": "common.ordered_diff",
     "cisco": "common.ordered_diff",
+    "iosxr": "common.ordered_diff",
     "nexus": "common.ordered_diff",
     "juniper": "juniper.ordered_diff",
     "arista": "common.ordered_diff",
@@ -49,6 +52,7 @@ VENDOR_EXIT = {
     "h3c": "quit",
     "optixtrans": "quit",
     "cisco": "exit",
+    "iosxr": "exit",
     "nexus": "exit",
     "arista": "exit",
     "juniper": "",

annet/annlib/rulebook/common.py

@@ -297,7 +297,7 @@ def apply(hw, do_commit, do_finalize, **_):
             after.add_cmd(Command("abort"))  # просто exit оставит висеть configure session
         if do_finalize:
             after.add_cmd(Command("write memory"))
-    elif hw.ASR or hw.XRV:
+    elif hw.ASR or hw.XRV or hw.XR:
         # коммит сам сохраняет изменения в стартап do_finalize не применим
         before.add_cmd(Command("configure exclusive"))
         if do_commit:

annet/api/__init__.py

@@ -48,6 +48,7 @@ from annet.parallel import Parallel, TaskResult
 from annet.reference import RefTracker
 from annet.storage import Device, get_storage
 from annet.types import Diff, ExitCode, OldNewResult, Op, PCDiff, PCDiffFile
+from annet.vendors import registry_connector
 
 DEFAULT_INDENT = "  "
 
@@ -786,7 +787,7 @@ def guess_hw(config_text: str):
     текста конфига и annet/rulebook/texts/*.rul"""
     scores = {}
     hw_provider = hardware_connector.get()
-    for vendor in VENDOR_REVERSES:
+    for vendor in registry_connector.get():
         hw = hw_provider.vendor_to_hw(vendor)
         fmtr = tabparser.make_formatter(hw)
         rb = rulebook.get_rulebook(hw)

annet/generators/__init__.py

@@ -197,7 +197,7 @@ def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRu
                 raise
             except Exception as err:
                 filename, lineno = gen.get_running_line()
-                logger.exception("Generator error in file '%s:%i'", filename, lineno)
+                logger.error("Generator error in file '%s:%i'", filename, lineno)
                 raise GeneratorError(f"{gen} on {device}") from err
 
             fmtr = tabparser.make_formatter(device.hw)

annet/hardware.py

@@ -1,17 +1,11 @@
 import abc
 from typing import Any
 
-from annet.annlib.netdev.views.hardware import HardwareView, hw_to_vendor
-
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.vendors import registry_connector
 from annet.connectors import Connector
 
 
-try:
-    from annet.annlib.netdev.views.hardware import vendor_to_hw
-except ImportError:
-    from netdev.views.hardware import vendor_to_hw
-
-
 class _HardwareConnector(Connector["HarwareProvider"]):
     name = "Hardware"
     ep_name = "hardware"
@@ -31,7 +25,7 @@ class HarwareProvider(abc.ABC):
         pass
 
     @abc.abstractmethod
-    def hw_to_vendor(self, hw: Any) -> str:
+    def hw_to_vendor(self, hw: Any) -> str | None:
         pass
 
 
@@ -40,7 +34,9 @@ class AnnetHardwareProvider(HarwareProvider):
         return HardwareView(hw_model, sw_version)
 
     def vendor_to_hw(self, vendor: str) -> HardwareView:
-        return vendor_to_hw(vendor)
+        return registry_connector.get().get(vendor.lower()).hardware
 
-    def hw_to_vendor(self, hw: HardwareView) -> str:
-        return hw_to_vendor(hw)
+    def hw_to_vendor(self, hw: HardwareView) -> str | None:
+        if vendor := registry_connector.get().match(hw, None):
+            return vendor.NAME
+        return None

annet/parallel.py

@@ -15,13 +15,14 @@ import tempfile
 import time
 import traceback
 import warnings
-from typing import Any, List, Optional, Type
+from typing import Any, List, Optional, Type, Callable
 from uuid import uuid4
 
 from contextlog import get_logger
 
 import annet
 from annet import tracing
+from annet.connectors import Connector
 from annet.lib import catch_ctrl_c, find_exc_in_stack
 from annet.output import capture_output
 from annet.tracing import tracing_connector
@@ -38,6 +39,20 @@ class PoolWorkerTask:
     payload: Optional[Any] = None
 
 
+class _PickleSafeTracebackFormatterConnector(Connector[Callable[[BaseException], str]]):
+    name = "PickleSafeTraceBackFormatter"
+    ep_name = "pickle_safe_traceback_formatter"
+
+    def _get_default(self) -> Callable[[], Callable[[BaseException], str]]:
+        def wrapper():
+            return lambda x: "".join(traceback.format_exception(x))
+
+        return wrapper
+
+
+pickle_safe_traceback_formatter_connector = _PickleSafeTracebackFormatterConnector()
+
+
 class PickleSafeException(Exception):
     """An exception that can be safely pickled and passed between processes."""
 
@@ -59,9 +74,14 @@ class PickleSafeException(Exception):
         pretty_lines.append(self.formatted_output)
         return "\n".join(pretty_lines)
 
-    @staticmethod
-    def from_exc(orig_exc: Exception, device_id: str, formatted_output: str) -> "PickleSafeException":
-        return PickleSafeException(orig_exc.__class__, str(orig_exc), device_id, formatted_output)
+    @classmethod
+    def from_exc(cls, orig_exc: Exception, device_id: str) -> "PickleSafeException":
+        return PickleSafeException(
+            orig_exc.__class__,
+            str(orig_exc),
+            device_id,
+            pickle_safe_traceback_formatter_connector.get()(orig_exc)
+        )
 
 
 @catch_ctrl_c
@@ -164,7 +184,7 @@ def _pool_worker(pool, index, task_queue, done_queue):
         except KeyboardInterrupt:  # pylint: disable=try-except-raise
             raise
         except Exception as exc:
-            safe_exc = PickleSafeException.from_exc(exc, task.payload, traceback.format_exc())
+            safe_exc = PickleSafeException.from_exc(exc, task.payload)
             ret_exc = safe_exc
             task_result.exc = safe_exc
             task_result.result = None
@@ -301,11 +321,10 @@ class Parallel:
                             **self.kwargs
                         )
                 except Exception as exc:
+                    safe_exc = PickleSafeException.from_exc(exc, device_id)
                     if not tolerate_fails:
-                        raise
-                    exc.device_id = device_id
-                    exc.formatted_output = traceback.format_exc()
-                    task_result.exc = exc
+                        raise safe_exc
+                    task_result.exc = safe_exc
                 if self.capture_output:
                     task_result.extra["cap_stdout"] = cap_stdout.read()
                     task_result.extra["cap_stderr"] = cap_stderr.read()

annet/rulebook/__init__.py

@@ -10,6 +10,7 @@ from annet.annlib.rbparser.platform import VENDOR_REVERSES, VENDOR_ALIASES
 from annet.connectors import CachedConnector
 from annet.rulebook.deploying import compile_deploying_text
 from annet.rulebook.patching import compile_patching_text
+from annet.vendors import registry_connector
 
 
 class RulebookProvider(ABC):
@@ -63,7 +64,7 @@ class DefaultRulebookProvider(RulebookProvider):
         if hw in self._rulebook_cache:
             return self._rulebook_cache[hw]
 
-        assert hw.vendor in VENDOR_REVERSES, "Unknown vendor: %s" % (hw.vendor)
+        assert hw.vendor in registry_connector.get(), "Unknown vendor: %s" % (hw.vendor)
         rul_vendor_name = VENDOR_ALIASES.get(hw.vendor, hw.vendor)
         patching = compile_patching_text(self._render_rul(rul_vendor_name + ".rul", hw), rul_vendor_name)