annet 0.0__py3-none-any.whl

annet/generators/exceptions.py

@@ -0,0 +1,10 @@
+class GeneratorError(Exception):
+    pass
+
+
+class InvalidValueFromGenerator(ValueError):
+    pass
+
+
+class NotSupportedDevice(GeneratorError):
+    pass

annet/generators/jsonfragment.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import contextlib
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Set,
+    Union,
+)
+
+from annet.storage import Device, Storage
+from .base import TreeGenerator, _filter_str
+from .exceptions import NotSupportedDevice
+
+
+class JSONFragment(TreeGenerator):
+    """Generates parts of JSON config file."""
+
+    TYPE = "JSON_FRAGMENT"
+    TAGS: List[str] = []
+
+    def __init__(self, storage: Storage):
+        super().__init__()
+        self.storage = storage
+        self._json_config: Dict[str, Any] = {}
+        self._config_pointer: List[str] = []
+
+        # if two generators edit same file, commands from generator with greater `reload_prio` will be used
+        if not hasattr(self, "reload_prio"):
+            self.reload_prio = 100
+
+    def supports_device(self, device: Device):
+        return bool(self.path(device))
+
+    def path(self, device: Device) -> Optional[str]:
+        raise NotImplementedError("Required PATH for JSON_FRAGMENT generator")
+
+    @classmethod
+    def get_aliases(cls) -> Set[str]:
+        return {cls.__name__, *cls.TAGS}
+
+    def acl(self, device: Device) -> Union[str, List[str]]:
+        """
+        Restrict the generator to a specified ACL using JSON Pointer syntax.
+
+        Expected ACL to be a list of strings, but a single string is also allowed.
+        """
+        raise NotImplementedError("Required ACL for JSON_FRAGMENT generator")
+
+    def acl_safe(self, device: Device) -> Union[str, List[str]]:
+        """
+        Restrict the generator to a specified safe ACL using JSON Pointer syntax.
+
+        Expected ACL to be a list of strings, but a single string is also allowed.
+        """
+        raise NotImplementedError("Required ACL for JSON_FRAGMENT generator")
+
+    def run(self, device: Device):
+        raise NotImplementedError
+
+    def get_reload_cmds(self, device: Device) -> str:
+        ret = self.reload(device) or ""
+        return ret
+
+    def reload(self, device) -> Optional[str]:
+        raise NotImplementedError
+
+    @contextlib.contextmanager
+    def block(self, *tokens, indent=None):  # pylint: disable=unused-argument
+        block_str = "".join(map(_filter_str, tokens))
+        self._config_pointer.append(block_str)
+        try:
+            yield
+        finally:
+            self._config_pointer.pop()
+
+    @contextlib.contextmanager
+    def block_piped(self, *tokens, indent=None):  # pylint: disable=unused-argument
+        block_str = "|".join(map(_filter_str, tokens))
+        self._config_pointer.append(block_str)
+        try:
+            yield
+        finally:
+            self._config_pointer.pop()
+
+    def __call__(self, device: Device, annotate: bool = False):
+        for cfg_fragment in self.run(device):
+            self._set_or_replace_dict(self._config_pointer, cfg_fragment)
+        return self._json_config
+
+    def _set_or_replace_dict(self, pointer, value):
+        if not pointer:
+            if self._json_config == {}:
+                self._json_config = value
+            else:
+                self._json_config = [self._json_config, value]
+        else:
+            self._set_dict(self._json_config, pointer, value)
+
+    @classmethod
+    def _to_str(cls, value: Any) -> str:
+        if isinstance(value, str):
+            return value
+        elif isinstance(value, list):
+            return [cls._to_str(x) for x in value]
+        elif isinstance(value, dict):
+            for k, v in value.items():
+                value[k] = cls._to_str(v)
+            return value
+        return str(value)
+
+    @classmethod
+    def _set_dict(cls, cfg, pointer, value):
+        # pointer has at least one key
+        if len(pointer) == 1:
+            if pointer[0] in cfg:
+                cfg[pointer[0]] = [cfg[pointer[0]], cls._to_str(value)]
+            else:
+                cfg[pointer[0]] = cls._to_str(value)
+        else:
+            if pointer[0] not in cfg:
+                cfg[pointer[0]] = {}
+            cls._set_dict(cfg[pointer[0]], pointer[1:], cls._to_str(value))

annet/generators/partial.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import re
+from typing import (
+    Iterable,
+    List,
+    Set,
+    Union,
+)
+
+from annet.lib import (
+    add_annotation,
+    flatten,
+)
+from .base import TreeGenerator, _filter_str
+from .exceptions import NotSupportedDevice
+
+
+class PartialGenerator(TreeGenerator):
+    TYPE = "PARTIAL"
+    TAGS: List[str] = []
+
+    def __init__(self, storage):
+        super().__init__()
+        self.storage = storage
+        self._annotate = False
+        self._running_gen = None
+        self._annotations = []
+        self._annotation_module = self.__class__.__module__ or ""
+
+    def supports_device(self, device) -> bool:
+        if self.__class__.run is PartialGenerator.run:
+            return bool(self._get_vendor_func(device.hw.vendor, "run"))
+        else:
+            return True
+
+    def acl(self, device):
+        acl_func = self._get_vendor_func(device.hw.vendor, "acl")
+        if acl_func:
+            return acl_func(device)
+
+    def acl_safe(self, device):
+        acl_func = self._get_vendor_func(device.hw.vendor, "acl_safe")
+        if acl_func:
+            return acl_func(device)
+
+    def run(self, device) -> Iterable[Union[str, tuple]]:
+        run_func = self._get_vendor_func(device.hw.vendor, "run")
+        if run_func:
+            return run_func(device)
+
+    def get_user_runner(self, device):
+        if self.__class__.run is not PartialGenerator.run:
+            return self.run
+        return self._get_vendor_func(device.hw.vendor, "run")
+
+    def _get_vendor_func(self, vendor: str, func_name: str):
+        attr_name = f"{func_name}_{vendor}"
+        if hasattr(self, attr_name):
+            return getattr(self, attr_name)
+        return None
+
+    # =====
+
+    @classmethod
+    def get_aliases(cls) -> Set[str]:
+        return {cls.__name__, *cls.TAGS}
+
+    def __call__(self, device, annotate=False):
+        self._indents = []
+        self._rows = []
+        self._running_gen = self.run(device)
+        self._annotate = annotate
+
+        if annotate and self.__class__.__module__:
+            self._annotation_module = ".".join(
+                self.__class__.__module__.split(".")[-2:])
+
+        for text in self._running_gen:
+            if isinstance(text, tuple):
+                text = " ".join(map(_filter_str, flatten(text)))
+            else:
+                text = _filter_str(text)
+            self._append_text(text)
+
+        for row in self._rows:
+            assert re.search(r"\bNone\b", row) is None, \
+                "Found 'None' in yield result: %s" % (row)
+        if annotate:
+            generated_rows = (
+                add_annotation(x, y)
+                for (x, y) in zip(self._rows, self._annotations)
+            )
+        else:
+            generated_rows = self._rows
+        return "\n".join(generated_rows) + "\n"
+
+    def _append_text(self, text):
+        def annotation_cb(row):
+            annotation = "%s:%d" % self.get_running_line()
+            self._annotations.append(annotation)
+            return row
+
+        self._append_text_cb(
+            text,
+            annotation_cb if self._annotate else None
+        )
+
+    def get_running_line(self):
+        if not self._running_gen or not self._running_gen.gi_frame:
+            return (repr(self._running_gen), -1)
+        return self._annotation_module, self._running_gen.gi_frame.f_lineno
+
+    @classmethod
+    def literal(cls, item):
+        return '"{}"'.format(item)
+
+    def __repr__(self):
+        return "<%s>" % self.__class__.__name__

annet/generators/perf.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import time
+from typing import (
+    Optional,
+    Union,
+)
+
+from annet import tracing
+from annet.tracing import tracing_connector
+from annet.types import (
+    GeneratorPartialRunArgs,
+    GeneratorPerf,
+)
+from .entire import Entire
+from .partial import PartialGenerator
+
+
+class GeneratorPerfMesurer:
+    def __init__(
+            self,
+            gen: Union[PartialGenerator, Entire],
+            run_args: Optional[GeneratorPartialRunArgs] = None,
+            trace_min_duration: tracing.MinDurationT = None
+    ):
+        self._gen = gen
+        self._run_args = run_args
+
+        self._start_time: float = 0.0
+        self._span_ctx = None
+        self._span = None
+        self._trace_min_duration = trace_min_duration
+
+        self.last_result: Optional[GeneratorPerf] = None
+
+    def start(self) -> None:
+        self.last_result = None
+
+        self._span_ctx = tracing_connector.get().start_as_current_span(
+            "gen:call",
+            tracer_name=self._gen.__class__.__module__,
+            min_duration=self._trace_min_duration,
+        )
+        self._span = self._span_ctx.__enter__()  # pylint: disable=unnecessary-dunder-call
+
+        if self._span:
+            self._span.set_attributes(
+                {"generator.class": self._gen.__class__.__name__})
+            if self._run_args:
+                tracing_connector.get().set_device_attributes(
+                    self._span, self._run_args.device,
+                )
+
+        self._start_time = time.monotonic()
+
+    def finish(self, exc_type=None, exc_val=None,
+               exc_tb=None) -> GeneratorPerf:
+        total = time.monotonic() - self._start_time
+        self._span_ctx.__exit__(exc_type, exc_val, exc_tb)
+
+        meta = {}
+        if tracing_connector.get().enabled:
+            span_context = self._span.get_span_context()
+            meta = {
+                "span": {
+                    "trace_id": str(span_context.trace_id),
+                    "span_id": str(span_context.span_id),
+                },
+            }
+
+        self.last_result = GeneratorPerf(total=total, rt=total, meta=meta)
+        return self.last_result
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.finish(exc_type, exc_val, exc_tb)

annet/generators/ref.py

@@ -0,0 +1,15 @@
+from .partial import PartialGenerator
+
+
+class RefGenerator(PartialGenerator):
+    def __init__(self, storage, groups=None):
+        super().__init__(storage)
+        self.groups = groups
+
+    def ref(self, device):
+        if hasattr(self, "ref_" + device.hw.vendor):
+            return getattr(self, "ref_" + device.hw.vendor)(device)
+        return ""
+
+    def with_groups(self, groups):
+        return type(self)(self.storage, groups)

annet/generators/result.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import textwrap
+from collections import OrderedDict as odict
+from typing import (
+    Any,
+    Dict,
+    Optional,
+    Tuple, Callable,
+)
+
+from annet.annlib import jsontools
+from annet.lib import (
+    merge_dicts,
+)
+from annet.reference import RefMatcher, RefTracker
+from annet.types import (
+    GeneratorEntireResult,
+    GeneratorJSONFragmentResult,
+    GeneratorPartialResult,
+)
+
+
+def _combine_acl_text(
+        partial_results: Dict[str, GeneratorPartialResult],
+        acl_getter: Callable[[GeneratorPartialResult], str]
+) -> str:
+    acl_text = ""
+    for gr in partial_results.values():
+        for line in textwrap.dedent(acl_getter(gr)).split("\n"):
+            if line and not line.isspace():
+                acl_text += line.rstrip()
+                acl_text += fr"  %generator_names={gr.name}"
+                acl_text += "\n"
+    return acl_text
+
+
+class RunGeneratorResult:
+    """
+    Результат запуска run_partial_generators/run_file_generators
+    """
+
+    def __init__(self):
+        self.partial_results: Dict[str, GeneratorPartialResult] = {}
+        self.entire_results: Dict[str, GeneratorEntireResult] = {}
+        self.json_fragment_results: Dict[str, GeneratorJSONFragmentResult] = {}
+        self.ref_track: RefTracker = RefTracker()
+        self.ref_matcher: RefMatcher = RefMatcher()
+
+    def add_partial(self, result: GeneratorPartialResult):
+        self.partial_results[result.name] = result
+
+    def add_entire(self, result: GeneratorEntireResult) -> None:
+        # Если есть несколько генераторов на один файл, выбрать тот, что с большим приоритетом
+        if result.path:
+            if result.path not in self.entire_results or \
+                    result.prio > self.entire_results[result.path].prio:
+                self.entire_results[result.path] = result
+
+    def add_json_fragment(self, result: GeneratorJSONFragmentResult) -> None:
+        self.json_fragment_results[result.name] = result
+
+    def config_tree(self, safe: bool = False) -> Dict[str, Any]:  # OrderedDict
+        tree = odict()
+        for gr in self.partial_results.values():
+            config = gr.safe_config if safe else gr.config
+            tree = merge_dicts(tree, config)
+        return tree
+
+    def new_files(self, safe: bool = False) -> Dict[str, Tuple[str, str]]:
+        files = {}
+        for gr in self.entire_results.values():
+            if not safe or gr.is_safe:
+                files[gr.path] = (gr.output, gr.reload)
+        return files
+
+    def acl_text(self) -> str:
+        return _combine_acl_text(self.partial_results, lambda gr: gr.acl)
+
+    def acl_safe_text(self) -> str:
+        return _combine_acl_text(self.partial_results, lambda gr: gr.acl_safe)
+
+    def new_json_fragment_files(
+            self,
+            old_files: Dict[str, Optional[str]],
+            safe: bool = False,
+    ) -> Dict[str, Tuple[Any, Optional[str]]]:
+        # TODO: safe
+        files: Dict[str, Tuple[Any, Optional[str]]] = {}
+        reload_prios: Dict[str, int] = {}
+        for generator_result in self.json_fragment_results.values():
+            filepath = generator_result.path
+            if filepath not in files:
+                if old_files.get(filepath) is not None:
+                    files[filepath] = (old_files[filepath], None)
+                else:
+                    files[filepath] = ({}, None)
+            result_acl = generator_result.acl
+            if safe:
+                result_acl = generator_result.acl_safe
+            previous_config: Dict[str, Any] = files[filepath][0]
+            new_fragment = generator_result.config
+            new_config = jsontools.apply_json_fragment(
+                previous_config,
+                new_fragment,
+                result_acl,
+            )
+            if filepath in reload_prios and \
+                    reload_prios[filepath] > generator_result.reload_prio:
+                _, reload_cmd = files[filepath]
+            else:
+                reload_cmd = generator_result.reload
+                reload_prios[filepath] = generator_result.reload_prio
+            files[filepath] = (new_config, reload_cmd)
+        return files
+
+    def perf_mesures(self) -> Dict[str, Dict[str, int]]:
+        mesures = {}
+        for gr in self.partial_results.values():
+            mesures[gr.name] = {"total": gr.perf.total,
+                                "rt": gr.perf.rt,
+                                "meta": gr.perf.meta}
+        for gr in self.entire_results.values():
+            mesures[gr.name] = {"total": gr.perf.total,
+                                "rt": gr.perf.rt,
+                                "meta": gr.perf.meta}
+        return mesures

annet/hardware.py

@@ -0,0 +1,45 @@
+import abc
+from typing import Any
+
+from annet.annlib.netdev.views.hardware import HardwareView, hw_to_vendor
+
+from annet.connectors import Connector
+
+
+try:
+    from annet.annlib.netdev.views.hardware import vendor_to_hw
+except ImportError:
+    from netdev.views.hardware import vendor_to_hw
+
+
+class _HardwareConnector(Connector["HarwareProvider"]):
+    name = "Hardware"
+    ep_name = "hardware"
+
+
+hardware_connector = _HardwareConnector()
+
+
+class HarwareProvider(abc.ABC):
+    @abc.abstractmethod
+    def make_hw(self, hw_model: str, sw_version: str) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def vendor_to_hw(self, vendor: str) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def hw_to_vendor(self, hw: Any) -> str:
+        pass
+
+
+class AnnetHardwareProvider(HarwareProvider):
+    def make_hw(self, hw_model: str, sw_version: str) -> HardwareView:
+        return HardwareView(hw_model, sw_version)
+
+    def vendor_to_hw(self, vendor: str) -> HardwareView:
+        return vendor_to_hw(vendor)
+
+    def hw_to_vendor(self, hw: HardwareView) -> str:
+        return hw_to_vendor(hw)

annet/implicit.py

@@ -0,0 +1,139 @@
+from collections import OrderedDict as odict
+
+from annet.annlib.rbparser import syntax
+
+
+def config(config_tree, rules):
+    implicit_config_tree = odict()
+    for (row, rule) in rules.items():
+        matched_lines = [line for line in config_tree.keys() if rule["regexp"].match(line)]
+        if rule["type"] != "ignore":
+            if not any(matched_lines) and row not in config_tree:
+                implicit_config_tree[row] = odict()
+        for line in matched_lines:
+            implicit_config_tree[line] = config(config_tree[line], rule["children"])
+    return implicit_config_tree
+
+
+def compile_rules(device):
+    return compile_tree(_implicit_tree(device))
+
+
+def compile_tree(tree):
+    rules = odict()
+    for (_, attrs) in tree.items():
+        rule = {
+            "type": attrs["type"],
+            "children": compile_tree(attrs["children"]) if attrs.get("children") else odict(),
+            "regexp": syntax.compile_row_regexp(attrs["row"]),
+        }
+        rules[attrs["row"]] = rule
+    return rules
+
+
+def _implicit_tree(device):
+    text = ""
+    if device.hw.Huawei:
+        if device.hw.Huawei.CE:
+            text = """
+                stp mode mstp
+                stp enable
+                undo ntp server disable
+                undo telnet server disable
+                undo dhcp enable
+                !user-interface con *
+                    user privilege level 3
+                !user-interface vty ~
+                    protocol inbound all
+                netconf
+            """
+        elif device.hw.Huawei.NE:
+            text = """
+                 !bgp *
+                     !ipv4-family unicast
+                         undo synchronization
+                     !ipv6-family unicast
+                         undo synchronization
+                 !user-interface con *
+                     user privilege level 3
+                 !user-interface vty ~
+                     protocol inbound all
+                 aaa
+                    undo user-password complexity-check
+                 netconf
+                 """
+        else:
+            text = """
+                stp mode mstp
+                !interface X?GigabitEthernet*
+                    bpdu enable
+                netconf
+            """
+    elif device.hw.Arista:
+        # эта часть конфигурации будет не видна в конфиге, если она включена с таким набором полей:
+        text = r"""
+                ip load-sharing trident fields ipv6 destination-port source-ip ingress-interface destination-ip source-port flow-label
+                ip load-sharing trident fields ip source-ip source-port destination-ip destination-port ingress-interface
+        """
+    elif device.hw.Nexus:
+        text = r"""
+                # часть конфигурации скрытая если включена
+                snmp-server enable traps link linkDown
+                snmp-server enable traps link linkUp
+        """
+        if device.hw.Nexus.N3x.N3432 \
+            or (device.hw.Nexus.N9x.N9500 and "spine1" in device.tags) \
+            or device.hw.Nexus.N9x.N9316 or device.hw.Cisco.Nexus.N9x.N9364:
+            text += r"""
+                # SVI
+                !interface Vlan*
+                    !shutdown
+                !interface mgmt[0-9]*
+                    no shutdown
+                    mtu 1500
+                !interface Ethernet1*
+                    no shutdown
+                # Лупбеки
+                !interface */Loopback[0-9.]+/
+                    no shutdown
+                # Агрегаты
+                !interface */port-channel[0-9.]+/
+                    no shutdown
+                # BGP
+                !router bgp *
+                    !neighbor *
+                        no shutdown
+            """
+
+        elif device.hw.Nexus.N3x:
+            # У нексуса сложные взаимоотношения с
+            # shutdown командой вездe
+            # в данный момент поведение проверенно для Cisco Nexus 3132Q 6.0(2)U6(7)
+            text += r"""
+                # SVI
+                !interface Vlan*
+                    !shutdown
+                !interface mgmt[0-9]*
+                    no shutdown
+                # Физические НЕ сплитованные интерфейсы и subif'ы
+                !interface */Ethernet1\/[0-9.]*/
+                    no shutdown
+                # Физические НЕ сплитованные интерфейсы и subif'ы
+                !interface */Ethernet1\/[0-9]+\/[0-9.]+/
+                    # только explicit
+                # Лупбеки
+                !interface */Loopback[0-9.]+/
+                    no shutdown
+                # Агрегаты
+                !interface */port-channel[0-9.]+/
+                    no shutdown
+                # BGP
+                !router bgp *
+                    !neighbor *
+                        no shutdown
+            """
+    return parse_text(text)
+
+
+def parse_text(text):
+    return syntax.parse_text(text, {})