annet 0.1__py3-none-any.whl

annet/generators/common/initial.py

@@ -0,0 +1,33 @@
+from annet.generators import PartialGenerator
+
+
+class InitialConfig(PartialGenerator):
+
+    """
+    Конфиги у свежих (еще ни разу не настраиваемых устройств)
+    на самом деле НЕ пустые. В данном генераторе отображен
+    такой набор команд, по крайней мере тех, которые могут
+    изменяться в ходе первичной конфигурации.
+
+    Acl для данного генератора не нужен, он будет генерировать
+    конфиг целиком.
+    """
+    def __init__(self, storage=None):
+        self._do_run = not storage
+        super().__init__(storage=None)
+
+    def run_huawei(self, device):
+        if not self._do_run:
+            return
+        if device.hw.CE:
+            yield """
+            telnet server disable
+            telnet ipv6 server disable
+            diffserv domain default
+            aaa
+                authentication-scheme default
+                authorization-scheme default
+                accounting-scheme default
+                domain default
+                domain default_admin
+            """

annet/hardware.py

@@ -0,0 +1,45 @@
+import abc
+from typing import Any
+
+from annet.annlib.netdev.views.hardware import HardwareView, hw_to_vendor
+
+from annet.connectors import Connector
+
+
+try:
+    from annet.annlib.netdev.views.hardware import vendor_to_hw
+except ImportError:
+    from netdev.views.hardware import vendor_to_hw
+
+
+class _HardwareConnector(Connector["HarwareProvider"]):
+    name = "Hardware"
+    ep_name = "hardware"
+
+
+hardware_connector = _HardwareConnector()
+
+
+class HarwareProvider(abc.ABC):
+    @abc.abstractmethod
+    def make_hw(self, hw_model: str, sw_version: str) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def vendor_to_hw(self, vendor: str) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def hw_to_vendor(self, hw: Any) -> str:
+        pass
+
+
+class AnnetHardwareProvider(HarwareProvider):
+    def make_hw(self, hw_model: str, sw_version: str) -> HardwareView:
+        return HardwareView(hw_model, sw_version)
+
+    def vendor_to_hw(self, vendor: str) -> HardwareView:
+        return vendor_to_hw(vendor)
+
+    def hw_to_vendor(self, hw: HardwareView) -> str:
+        return hw_to_vendor(hw)

annet/implicit.py

@@ -0,0 +1,139 @@
+from collections import OrderedDict as odict
+
+from annet.annlib.rbparser import syntax
+
+
+def config(config_tree, rules):
+    implicit_config_tree = odict()
+    for (row, rule) in rules.items():
+        matched_lines = [line for line in config_tree.keys() if rule["regexp"].match(line)]
+        if rule["type"] != "ignore":
+            if not any(matched_lines) and row not in config_tree:
+                implicit_config_tree[row] = odict()
+        for line in matched_lines:
+            implicit_config_tree[line] = config(config_tree[line], rule["children"])
+    return implicit_config_tree
+
+
+def compile_rules(device):
+    return compile_tree(_implicit_tree(device))
+
+
+def compile_tree(tree):
+    rules = odict()
+    for (_, attrs) in tree.items():
+        rule = {
+            "type": attrs["type"],
+            "children": compile_tree(attrs["children"]) if attrs.get("children") else odict(),
+            "regexp": syntax.compile_row_regexp(attrs["row"]),
+        }
+        rules[attrs["row"]] = rule
+    return rules
+
+
+def _implicit_tree(device):
+    text = ""
+    if device.hw.Huawei:
+        if device.hw.Huawei.CE:
+            text = """
+                stp mode mstp
+                stp enable
+                undo ntp server disable
+                undo telnet server disable
+                undo dhcp enable
+                !user-interface con *
+                    user privilege level 3
+                !user-interface vty ~
+                    protocol inbound all
+                netconf
+            """
+        elif device.hw.Huawei.NE:
+            text = """
+                 !bgp *
+                     !ipv4-family unicast
+                         undo synchronization
+                     !ipv6-family unicast
+                         undo synchronization
+                 !user-interface con *
+                     user privilege level 3
+                 !user-interface vty ~
+                     protocol inbound all
+                 aaa
+                    undo user-password complexity-check
+                 netconf
+                 """
+        else:
+            text = """
+                stp mode mstp
+                !interface X?GigabitEthernet*
+                    bpdu enable
+                netconf
+            """
+    elif device.hw.Arista:
+        # эта часть конфигурации будет не видна в конфиге, если она включена с таким набором полей:
+        text = r"""
+                ip load-sharing trident fields ipv6 destination-port source-ip ingress-interface destination-ip source-port flow-label
+                ip load-sharing trident fields ip source-ip source-port destination-ip destination-port ingress-interface
+        """
+    elif device.hw.Nexus:
+        text = r"""
+                # часть конфигурации скрытая если включена
+                snmp-server enable traps link linkDown
+                snmp-server enable traps link linkUp
+        """
+        if device.hw.Nexus.N3x.N3432 \
+            or (device.hw.Nexus.N9x.N9500 and "spine1" in device.tags) \
+            or device.hw.Nexus.N9x.N9316 or device.hw.Cisco.Nexus.N9x.N9364:
+            text += r"""
+                # SVI
+                !interface Vlan*
+                    !shutdown
+                !interface mgmt[0-9]*
+                    no shutdown
+                    mtu 1500
+                !interface Ethernet1*
+                    no shutdown
+                # Лупбеки
+                !interface */Loopback[0-9.]+/
+                    no shutdown
+                # Агрегаты
+                !interface */port-channel[0-9.]+/
+                    no shutdown
+                # BGP
+                !router bgp *
+                    !neighbor *
+                        no shutdown
+            """
+
+        elif device.hw.Nexus.N3x:
+            # У нексуса сложные взаимоотношения с
+            # shutdown командой вездe
+            # в данный момент поведение проверенно для Cisco Nexus 3132Q 6.0(2)U6(7)
+            text += r"""
+                # SVI
+                !interface Vlan*
+                    !shutdown
+                !interface mgmt[0-9]*
+                    no shutdown
+                # Физические НЕ сплитованные интерфейсы и subif'ы
+                !interface */Ethernet1\/[0-9.]*/
+                    no shutdown
+                # Физические НЕ сплитованные интерфейсы и subif'ы
+                !interface */Ethernet1\/[0-9]+\/[0-9.]+/
+                    # только explicit
+                # Лупбеки
+                !interface */Loopback[0-9.]+/
+                    no shutdown
+                # Агрегаты
+                !interface */port-channel[0-9.]+/
+                    no shutdown
+                # BGP
+                !router bgp *
+                    !neighbor *
+                        no shutdown
+            """
+    return parse_text(text)
+
+
+def parse_text(text):
+    return syntax.parse_text(text, {})

annet/lib.py

@@ -0,0 +1,128 @@
+import asyncio
+import os
+import shutil
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Awaitable, Optional
+
+import yaml
+from annet.annlib.lib import (  # pylint: disable=unused-import
+    ContextOrderedDict,
+    HuaweiNumBlock,
+    LMSMatcher,
+    add_annotation,
+    catch_ctrl_c,
+    cisco_collapse_vlandb,
+    cisco_expand_vlandb,
+    find_exc_in_stack,
+    find_modules,
+    first,
+    flatten,
+    huawei_collapse_vlandb,
+    huawei_expand_vlandb,
+    huawei_iface_ranges,
+    is_relative,
+    jinja_render,
+    jun_activate,
+    jun_is_inactive,
+    juniper_fmt_prefix_lists_acl,
+    juniper_port_split,
+    make_ip4_mask,
+    mako_render,
+    merge_dicts,
+    percentile,
+    uniq,
+)
+from contextlog import get_logger
+
+
+_TEMPLATE_CONTEXT_PATH: Optional[str] = None
+_DEFAULT_CONTEXT_PATH: Optional[str] = None
+
+
+def get_template_context_path() -> str:
+    if _TEMPLATE_CONTEXT_PATH is None:
+        set_template_context_path(str(Path(sys.modules["annet"].__file__).parent / "configs/context.yml"))
+    return _TEMPLATE_CONTEXT_PATH
+
+
+def set_template_context_path(path: str) -> None:
+    global _TEMPLATE_CONTEXT_PATH  # pylint: disable=global-statement
+    _TEMPLATE_CONTEXT_PATH = path
+
+
+def get_default_context_path() -> str:
+    if _DEFAULT_CONTEXT_PATH is None:
+        set_default_context_path("~/.annet/context.yml")
+    return _DEFAULT_CONTEXT_PATH
+
+
+def set_default_context_path(path: str) -> None:
+    global _DEFAULT_CONTEXT_PATH  # pylint: disable=global-statement
+    _DEFAULT_CONTEXT_PATH = path
+
+
+@lru_cache(maxsize=1)
+def _get_template_context():
+    with open(get_template_context_path()) as f:
+        return yaml.safe_load(f)
+
+
+def get_context_path(touch: Optional[bool] = False) -> str:
+    path = Path(os.getenv("ANN_CONTEXT_CONFIG_PATH", get_default_context_path())).expanduser().absolute()
+    if not path.exists():
+        src = get_template_context_path()
+        if not touch:
+            return str(src)
+        try:
+            # populate path with default configuration
+            path.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(src, path)
+        except shutil.SameFileError:
+            pass
+    return str(path)
+
+
+def get_context() -> dict:
+    with open(get_context_path()) as f:
+        raw = yaml.safe_load(f)
+    _fill_in_default_generator_modules(raw)
+    context_name = os.getenv("ANN_SELECTED_CONTEXT", raw["selected_context"])
+    res = {k: raw[k][v] for k, v in raw["context"][context_name].items()}
+    if "ANN_GENERATORS_CONTEXT" in os.environ:  # an undocumented hack to maintain backwards compatibility; TODO: remove
+        res["generators"] = raw["generators"][os.getenv("ANN_GENERATORS_CONTEXT")]
+    return res
+
+
+@lru_cache(maxsize=1)
+def _warn_no_generators_in_context():
+    get_logger().warning(
+        "Older version of the context configuration found. Getting generators references from the template context"
+    )
+
+
+def _fill_in_default_generator_modules(raw: dict) -> bool:
+    """Backwards compatibility hack to add existing generators refs to context"""
+    if "generators" not in raw:
+        _warn_no_generators_in_context()
+        raw["generators"] = _get_template_context()["generators"]
+        for dst_context in raw["context"].values():
+            dst_context["generators"] = "default"
+        return True
+    return False
+
+
+def repair_context_file() -> None:
+    path = get_context_path()
+    with open(path) as f:
+        data = yaml.safe_load(f)
+    if _fill_in_default_generator_modules(data):
+        with open(path, "w") as f:
+            yaml.dump(data, f, sort_keys=False)
+
+
+def do_async(coro: Awaitable):
+    loop = asyncio.get_event_loop()
+    res = loop.run_until_complete(coro)
+    return res

annet/output.py

@@ -0,0 +1,170 @@
+import abc
+import os
+import posixpath
+import sys
+from typing import List, Optional, Tuple, Type
+
+import colorama
+from annet.annlib.output import (  # pylint: disable=unused-import
+    LABEL_NEW_PREFIX,
+    OutputWriter,
+    TextArgs,
+    capture_output,
+    dir_or_file_output,
+    format_file_diff,
+    print_as_json,
+    print_as_yaml,
+    print_err_label,
+    print_label,
+)
+from contextlog import get_logger
+
+from annet.cli_args import FileOutOptions, QueryOptions
+from annet.connectors import Connector
+from annet.storage import Device, storage_connector
+
+
+class _DriverConnector(Connector["OutputDriver"]):
+    name = "OutputDriver"
+    ep_name = "output"
+
+    def _get_default(self) -> Type["OutputDriver"]:
+        return OutputDriverBasic
+
+
+output_driver_connector = _DriverConnector()
+
+
+class OutputDriver(abc.ABC):
+    @abc.abstractmethod
+    def write_output(self, arg_out: FileOutOptions, items, query_result_count=1) -> None:
+        pass
+
+    @abc.abstractmethod
+    def format_fails(self, fail, args: Optional[QueryOptions] = None) -> Tuple[str, str]:
+        pass
+
+    @abc.abstractmethod
+    def cfg_file_names(self, device: Device) -> List[str]:
+        pass
+
+    @abc.abstractmethod
+    def entire_config_dest_path(self, device: Device, config_path: str) -> str:
+        pass
+
+
+class OutputDriverBasic(OutputDriver):
+    def write_output(self, arg_out: FileOutOptions, items, query_result_count=1):
+        """
+        пишет результаты генерации в файл или директорию :dest
+        :dest - это директория в случаях:
+        - заканчивается на "/"
+        - существует директория с таким именем
+        - более одного устройства в результате запроса
+        - есть entire-генераторы (определяется по типу первого результата, если устройство одно)
+        """
+        logger = get_logger()
+
+        items_iter = iter(items)
+        try:
+            first_result = next(items_iter)
+        except StopIteration:
+            # нет результатов, ничего не пишем и не создаём
+            return
+
+        def _reassemble_items():
+            yield first_result
+            yield from items_iter
+
+        dest = arg_out.dest
+        dir_mode = dir_or_file_output(dest, query_result_count, suggest_dir=(os.sep in first_result[0]))
+        _reassembled_items = list(_reassemble_items())
+
+        for output_no, (label, output, is_fail) in enumerate(_reassembled_items):
+            writer = output if isinstance(output, OutputWriter) else OutputWriter(output)
+            label = os.path.normpath(label)
+            label_color = colorama.Back.RED if is_fail else colorama.Back.GREEN
+            if dest is None:
+                if hasattr(arg_out, "format") and arg_out.format == "json":
+                    if output_no > 0:
+                        sys.stdout.write(",")
+                    elif output_no == 0:
+                        sys.stdout.write("{")
+                    sys.stdout.write('"%s": ' % label)
+                    writer.write(sys.stdout)
+                    if output_no == len(_reassembled_items) - 1:
+                        sys.stdout.write("}")
+                else:
+                    if not arg_out.no_label:
+                        print_label(label, back_color=label_color)
+                    writer.write(sys.stdout)
+            elif dir_mode:
+                if label.startswith(LABEL_NEW_PREFIX):
+                    label = label[len(LABEL_NEW_PREFIX):]
+                if label.startswith(os.sep):
+                    # just in case.
+                    label = label.lstrip(os.sep)
+                if os.sep not in label:
+                    # vendor config
+                    label = os.path.basename(label)
+                else:
+                    # entire generated file
+                    parts = label.split(os.sep)
+                    hostname = parts[0]
+                    label = os.sep.join(parts[1:])
+                    if not arg_out.expand_path:
+                        label = os.path.basename(label)
+                    if query_result_count > 1:
+                        label = os.path.join(hostname, label)
+                file_dest = os.path.join(dest, "errors") if is_fail else dest
+                out_file = os.path.normpath(os.path.join(file_dest, label))
+                logger.info("writing '%s'", out_file)
+                dirname = os.path.dirname(out_file)
+                if not os.path.exists(dirname):
+                    os.makedirs(dirname)
+                with open(out_file, "w") as file:
+                    writer.write(file)
+            else:
+                logger.info("writing '%s'", dest)
+                with open(dest, "w") as file:
+                    writer.write(file)
+
+    def format_fails(self, fail, args: Optional[QueryOptions] = None):
+        ret = []
+        fqdns = {}
+        if args:
+            with storage_connector.get().storage()(args) as storage:
+                fqdns = storage.resolve_fdnds_by_query(args.query)
+        for (assignment, exc) in fail.items():
+            label = assignment
+            if assignment in fqdns:
+                label = fqdns[assignment]
+            elif isinstance(assignment, tuple):
+                label = assignment[0]
+            else:
+                ValueError("Failed to parse failed assignment %r" % assignment)
+            ret.append((label, getattr(exc, "formatted_output", f"{repr(exc)} (formatted_output is absent)"), True))
+        return ret
+
+    def cfg_file_names(self, device: Device) -> List[str]:
+        return [f"{device.hostname}.cfg"]
+
+    def entire_config_dest_path(self, device, config_path: str) -> str:
+        """Формирует путь к конфигу в директории destname.
+
+        Например, для устройства с hostname `my-device`:
+        ```
+        >>> device.entire_config_dest_path("/etc/frr/frr.conf")
+        'my-device.cfg/etc/frr/frr.conf'
+        >>>
+        ```
+        """
+        # NOTE: с полученным `config_path` работаем через `posixpath`, а не через `os.path`, потому что
+        #       entire-путь POSIX-специфичный; но в конце формируем путь через `os.path` для текущей платформы
+        if not posixpath.abspath(config_path):
+            raise RuntimeError(f"Want absolute config path, but relative received: {config_path}")
+        cfg_files = self.cfg_file_names(device)
+        # NOTE: получаем путь без "/" в начале, например, "etc/frr/frr.conf"
+        relative_config_path = posixpath.relpath(config_path, "/")
+        dest_config_path_parts = [cfg_files[0]] + relative_config_path.split(posixpath.sep)
+        return os.path.join(*dest_config_path_parts)