annet 0.1__py3-none-any.whl

annet/__init__.py

@@ -0,0 +1,61 @@
+import logging
+import logging.config
+import os
+import pkgutil
+import sys
+from argparse import SUPPRESS, Namespace
+
+import colorama
+import yaml
+from annet.annlib.errors import (  # pylint: disable=wrong-import-position
+    DeployCancelled,
+    ExecError,
+)
+from contextlog import patch_logging, patch_threading
+from valkit.python import valid_logging_level
+
+import annet.argparse
+
+
+__all__ = ("DeployCancelled", "ExecError")
+
+DEBUG2_LEVELV_NUM = 9
+
+
+def fill_base_args(parser: annet.argparse.ArgParser, pkg_name: str, logging_config: str):
+    parser.add_argument("--log-level", default="WARN", type=valid_logging_level,
+                        help="Уровень детализации логов (DEBUG, DEBUG2 (with comocutor debug), INFO, WARN, CRITICAL)")
+    parser.add_argument("--pkg_name", default=pkg_name, help=SUPPRESS)
+    parser.add_argument("--logging_config", default=logging_config, help=SUPPRESS)
+
+
+def init_logging(options: Namespace):
+    patch_logging()
+    patch_threading()
+    logging.captureWarnings(True)
+    logging_config = yaml.safe_load(pkgutil.get_data(options.pkg_name, options.logging_config))
+    if options.log_level is not None:
+        logging_config.setdefault("root", {})
+        logging_config["root"]["level"] = options.log_level
+    logging.addLevelName(DEBUG2_LEVELV_NUM, "DEBUG2")
+    logging.config.dictConfig(logging_config)
+
+
+def init(options: Namespace):
+    init_logging(options)
+
+    # Отключить colorama.init, если стоит env-переменная. Нужно в тестах
+    if os.environ.get("ANN_FORCE_COLOR", None) not in [None, "", "0", "no"]:
+        colorama.init = lambda *_, **__: None
+    colorama.init()
+
+    # Workaround for Python 3.8.0: https://bugs.python.org/issue38529
+    import asyncio.streams
+    if hasattr(asyncio.streams.StreamReaderProtocol, "_on_reader_gc"):
+        asyncio.streams.StreamReaderProtocol._on_reader_gc = lambda *args, **kwargs: None  # pylint: disable=protected-access
+
+
+def assert_python_version():
+    if sys.version_info < (3, 8, 0):
+        sys.stderr.write("Error: you need python 3.8.0 or higher\n")
+        sys.exit(1)

annet/annet.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import sys
+
+import annet
+from annet import argparse, cli, generators, hardware, lib, rulebook
+
+
+# =====
+@lib.catch_ctrl_c
+def main():
+    annet.assert_python_version()
+    parser = argparse.ArgParser()
+    cli.fill_base_args(parser, annet.__name__, "configs/logging.yaml")
+    rulebook.rulebook_provider_connector.set(rulebook.DefaultRulebookProvider)
+    hardware.hardware_connector.set(hardware.AnnetHardwareProvider)
+
+    parser.add_commands(parser.find_subcommands(cli.list_subcommands()))
+    try:
+        return parser.dispatch(pre_call=annet.init, add_help_command=True)
+    except (generators.GeneratorError, annet.ExecError):
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

annet/annlib/__init__.py

@@ -0,0 +1,7 @@
+import os
+
+import colorama
+
+# отключить colorama.init, если стоит env-переменная. Нужно в тестах
+if os.environ.get("ANN_FORCE_COLOR", None) not in [None, "", "0", "no"]:
+    colorama.init = lambda *_, **__: None

annet/annlib/command.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass, field
+
+from typing import List, Optional
+
+
+FIRST_EXCEPTION = 1
+ALL_COMPLETED = 2
+
+
+@dataclass
+class Question:
+    question: str  # frame it using / if it is a regular expression
+    answer: str
+    is_regexp: Optional[bool] = False
+
+
+@dataclass
+class Command:
+    cmd: str
+    questions: Optional[List[Question]] = None
+    exc_handler: Optional[List[Question]] = None
+    timeout: Optional[int] = None
+    suppress_nonzero: bool = False
+    suppress_eof: bool = False
+
+    def __str__(self) -> str:
+        return self.cmd
+
+
+@dataclass
+class CommandList:
+    cmss: List[Command] = field(default_factory=list)
+
+    def __post_init__(self):
+        if not self.cmss:
+            self.cmss = []
+
+    def __iter__(self):
+        return iter(self.cmss)
+
+    def __len__(self) -> int:
+        return len(self.cmss)
+
+    def add_cmd(self, cmd: Command) -> None:
+        assert isinstance(cmd, Command)
+        self.cmss.append(cmd)
+
+    def as_list(self) -> List[Command]:  # TODO: delete
+        return self.cmss

annet/annlib/diff.py

@@ -0,0 +1,158 @@
+import functools
+import ipaddress
+from typing import Dict, Generator
+
+import colorama
+
+from .types import Diff, Op
+
+# NOCDEV-1720
+
+
+diff_ops = {
+    "+": Op.ADDED,
+    "-": Op.REMOVED,
+    " ": Op.AFFECTED,
+    ">": Op.MOVED,
+}
+
+ops_sign = {
+    v: k
+    for k, v in diff_ops.items()
+}
+
+ops_order = {
+    Op.AFFECTED: 0,
+    Op.MOVED: 1,
+    Op.REMOVED: 2,
+    Op.ADDED: 3,
+}
+
+ops_color = {
+    Op.REMOVED: colorama.Fore.RED,
+    Op.ADDED: colorama.Fore.GREEN,
+    Op.AFFECTED: colorama.Fore.CYAN,
+    Op.MOVED: colorama.Fore.YELLOW,
+}
+
+
+def is_int(ts):
+    try:
+        int(ts)
+        return True
+    except ValueError:
+        return False
+
+
+def is_ip(ts):
+    try:
+        ipaddress.ip_interface(ts)
+        return True
+    except ValueError:
+        return False
+
+
+def diff_cmp(diff_l, diff_r):
+    """
+Сборник костылей для сравнения двух строк диффа
+    """
+    (op_l, line_l, _, _) = diff_l
+    (op_r, line_r, _, _) = diff_r
+
+    cmp_line = (line_l > line_r) - (line_l < line_r)
+    cmp_op = ops_order[op_l] - ops_order[op_r]
+    if cmp_line == 0:
+        # При равенстве строк порядок определяется операцией
+        return cmp_op
+
+    if cmp_op == 0:
+        # Если для строк операции одинаковы, то считаем их равными
+        # По идее TimSort стабилен и порядок просто не должен измениться
+        return 0
+
+    # Для частично совпадающих строк
+    lws = line_l.split(" ")
+    rws = line_r.split(" ")
+    res = 0
+    for i, lw in enumerate(lws):
+        if len(rws) > i:
+            rw = rws[i]
+            if is_int(lw) and is_int(rw):
+                # В одинаковом положении в строках есть инты, сортируем по ним
+                res = int(lw) - int(rw)
+                if res == 0:
+                    # При равных интах - сортируем по операции
+                    res = cmp_op
+            elif is_ip(lw) and is_ip(rw):
+                # Аналогично интам обрабатываем ip-адреса
+                ip_l = ipaddress.ip_interface(lw)
+                ip_r = ipaddress.ip_interface(rw)
+                try:
+                    res = (ip_l > ip_r) - (ip_l < ip_r)
+                except TypeError:
+                    res = 1
+                    if ip_l.version == 4:
+                        res = -1
+                if res == 0:
+                    res = cmp_op
+            elif i > 0:
+                if lw == rw:
+                    res = cmp_op
+            else:
+                continue
+            break
+    if res != 0:
+        return res
+    return cmp_line
+
+
+def resort_diff(diff: Diff) -> Diff:
+    res = []
+    df = sorted(diff, key=functools.cmp_to_key(diff_cmp))
+    for line in df:
+        ln = line
+        if len(line[2]) > 0:
+            ln = (line[0], line[1], resort_diff(line[2]), line[3])
+        res.append(ln)
+    return res
+
+
+def colorize_line_with_color(line: str, color: int, no_color: bool):
+    stripped = line.rstrip("\n")
+    add_newlines = len(line) - len(stripped)
+    line = stripped
+
+    if not no_color:
+        line = "%s%s%s%s" % (colorama.Style.BRIGHT, color, line, colorama.Style.RESET_ALL)
+
+    line += "\n" * add_newlines
+    return line
+
+
+def colorize_line(line, no_color=False):
+    op = diff_ops[line[0]]
+    color = ops_color[op]
+    return colorize_line_with_color(line, color, no_color)
+
+
+def gen_pre_as_diff(
+    pre: Dict,
+    show_rules: bool,
+    indent: str,
+    no_color: bool,
+    _level: int = 0
+) -> Generator[str, None, None]:
+    ops = [(order, op) for op, order in ops_order.items()]
+    ops.sort()
+    for (raw_rule, content) in pre.items():
+        items = content["items"].items()
+        for (_, diff) in items:  # pylint: disable=redefined-outer-name
+            if show_rules and not raw_rule == "__MULTILINE_BODY__":
+                line = "# %s%s\n" % (indent * _level, raw_rule)
+                yield colorize_line_with_color(line, colorama.Fore.BLACK, no_color)
+            for (op, rows) in [(op, diff[op]) for (_, op) in ops]:
+                for item in rows:
+                    line = "%s%s %s\n" % (ops_sign[op], indent * _level, item["row"])
+                    yield colorize_line_with_color(line, ops_color[op], no_color)
+                    if len(item["children"]) != 0:
+                        yield from gen_pre_as_diff(item["children"], show_rules, indent, no_color, _level + 1)

annet/annlib/errors.py

@@ -0,0 +1,8 @@
+class ExecError(Exception):
+    """Обработчик этого exception должен залоггировать ошибку и выйти с exit_code 1"""
+    pass
+
+
+class DeployCancelled(Exception):
+    """Деплой на устройство был отменен"""
+    pass

annet/annlib/filter_acl.py

@@ -0,0 +1,196 @@
+import collections
+import re
+import typing
+
+from . import patching, tabparser
+from .diff import diff_ops, ops_sign
+from .rbparser import acl
+
+UnifiedInputConfig = str  # Конфиг классических сетевых устройств
+FileInputConfig = typing.Dict[str, typing.Any]  # Конфиг вайтбоксов и серверов
+InputConfig = typing.Union[UnifiedInputConfig, FileInputConfig]
+
+Acl = typing.Dict[str, typing.Any]
+
+UnifiedConfigTree = typing.OrderedDict[str, typing.Any]
+FileConfigTree = typing.Dict[str, typing.Any]
+ConfigTree = typing.Union[UnifiedConfigTree, FileConfigTree]
+
+DiffTree = typing.OrderedDict[str, typing.Any]
+UnifiedDiff = typing.List[typing.Tuple[str, str, typing.List, typing.Optional[int]]]
+FileConfigDiff = typing.Dict[str, typing.Any]
+Diff = typing.Union[UnifiedDiff, FileConfigDiff]
+
+
+def make_acl(text: str, vendor: str) -> Acl:
+    return acl.compile_acl_text(text, vendor)
+
+
+def filter_config(acl: Acl, fmtr: tabparser.CommonFormatter, input_config: InputConfig) -> InputConfig:
+    if isinstance(input_config, str):
+        config: ConfigTree = tabparser.parse_to_tree(input_config, fmtr.split)
+        config = patching.apply_acl(config, acl, fatal_acl=False)
+        config = fmtr.join(config)
+    else:
+        config = typing.cast(input_config, FileConfigTree)
+        config = apply_acl_fileconfig(input_config, acl)
+    return config
+
+
+def filter_diff(acl: Acl, fmtr: tabparser.CommonFormatter, input_config: InputConfig) -> InputConfig:
+    if isinstance(input_config, str):
+        input_config = shift_op(input_config)
+        diff_tee: DiffTree = tabparser.parse_to_tree(input_config, fmtr.split)
+        diff: Diff = tree_to_diff(diff_tee)
+        diff = patching.apply_acl_diff(diff, acl)
+        config = fmtr.join(diff_to_tree(diff))
+        config = unshift_op(config)
+        config = config.rstrip()
+    else:
+        config = typing.cast(input_config, FileConfigTree)
+        config = apply_acl_fileconfig(input_config, acl)
+    return config
+
+
+def filter_patch(acl: Acl, fmtr: tabparser.CommonFormatter, text: str) -> str:
+    return filter_config(acl, fmtr, text)
+
+
+# NOCDEV-6378 на патч для Juniper/Nokia нельзя просто так наложить filter_acl
+def filter_patch_jun_nokia(diff_filtered: InputConfig, fmtr: tabparser.CommonFormatter, text: str) -> str:
+    """
+    Накладываем ACL на патчи для Juniper/Nokia
+
+    Поскольку в патче уже потерена иерархия команд - они развернуты в строки типа
+    Нужна дополнительная информация о изначальном конфиге, которую можно подсмотреть в дифе
+        set interface et-0/0/0 unit ....
+        delete interface et-0/0/0 unit ....
+        /configure port 1/1/c17/1 ...
+        /configure delete port 1/1/c18/1 ...
+    """
+    diff_tree_stripped: DiffTree = tabparser.parse_to_tree(strip_op(diff_filtered), fmtr.split)
+    _tree_expand_lists_nokia_jun(diff_tree_stripped)
+    patch_lines_passed = []
+    for patch_line in text.split("\n"):
+        patch_parts = [x for x in patch_line.split(" ") if x]
+        diff_current = diff_tree_stripped
+        # strip set|delete|/configure
+        while patch_parts and patch_parts[0] in {"/configure", "set", "delete"}:
+            patch_parts = patch_parts[1:]
+        while patch_parts and diff_current:
+            for i in range(len(patch_parts), -1, -1):
+                key = " ".join(patch_parts[:i])
+                # consume parts and go down in diff hierarchy
+                if key in diff_current:
+                    patch_parts = patch_parts[i:]
+                    diff_current = diff_current[key]
+                    break
+            # no progress has been made
+            else:
+                break
+        if not patch_parts:
+            patch_lines_passed.append(patch_line)
+    return "\n".join(patch_lines_passed)
+
+
+def apply_acl_fileconfig(config, rules):
+    passed = {}
+    for (filename, filecontent) in config.items():
+        (match, _) = patching.match_row_to_acl(filename, rules)
+        if match:
+            if not (match["is_reverse"] and match["attrs"]["cant_delete"]):
+                passed[filename] = filecontent
+    return passed
+
+
+def get_op(line: str) -> typing.Tuple[str, str, str]:
+    op = " "
+    indent = ""
+    opidx = -1
+    rowstart = 0
+    for rowstart in range(len(line)):
+        if line[rowstart] not in diff_ops:
+            break
+    for opidx in range(rowstart):
+        if line[opidx] != " ":
+            break
+    if opidx >= 0:
+        op = line[opidx]
+        indent = line[:opidx] + line[opidx+1:rowstart]
+    if op != " ":
+        indent = indent + " "
+    return op, indent, line[rowstart:]
+
+
+def shift_op(text: str) -> str:
+    ret = ""
+    for line in text.split("\n"):
+        op, indent, line = get_op(line)
+        ret += indent + op + line + "\n"
+    return ret
+
+
+def unshift_op(text: str) -> str:
+    ret = ""
+    for line in text.split("\n"):
+        op, indent, line = get_op(line)
+        ret += op + indent + line + "\n"
+    return ret
+
+
+def strip_op(text: str) -> str:
+    ret: str = ""
+    for line in text.split("\n"):
+        op, indent, line = get_op(line)
+        if op != " ":
+            indent = indent[1:]
+        ret += indent + line + "\n"
+    return ret
+
+
+def tree_to_diff(diff_tree: ConfigTree) -> Diff:
+    ret = []
+    for row, v in diff_tree.items():
+        op, _, row = get_op(row)
+        diff_op = diff_ops[op]
+        children = []
+        d_match = None
+        if isinstance(v, dict):
+            children = tree_to_diff(v)
+        ret.append((diff_op, row, children, d_match))
+    return ret
+
+
+def diff_to_tree(diff: Diff) -> ConfigTree:
+    ret = collections.OrderedDict()
+    for diff_op, row, children, _ in diff:
+        row = ops_sign[diff_op] + row
+        ret[row] = diff_to_tree(children)
+    return ret
+
+
+def _tree_expand_lists_nokia_jun(diff_tree: DiffTree):
+    """
+    Раскрываем списки Nokia/Juniper в отдельные элементы
+    {command: {"[a, b, c]": {}}}   ->   {command a: {}, command b: {}, command c: {}}
+
+    В неупорядоченном множестве префиксов также стираем ';' на конце - их не бывает в патче
+    {prefix-list: {"2a02::/64;": {}, "2a03::/64;": {}}}   ->   {prefix-list: {"2a02::/64": {}, "2a03::/64": {}}}
+    """
+    process: typing.List[DiffTree] = [diff_tree]
+    list_regexp = re.compile(r"^(.*)\s+\[(.+)\]$")
+    while process:
+        tree, process = process[0], process[1:]
+        for cmd in list(tree.keys()):
+            children = tree[cmd]
+            matches = list_regexp.search(cmd)
+            normalized_cmd = cmd.rstrip(";")
+            if matches:
+                for c in matches.group(2).split(" "):
+                    if c.strip():
+                        tree[" ".join([matches.group(1), c])] = children
+            if normalized_cmd != cmd:
+                del tree[cmd]
+                tree[normalized_cmd] = children
+            if isinstance(children, dict):
+                process.append(children)

annet/annlib/jsontools.py

@@ -0,0 +1,89 @@
+"""Support JSON patch (RFC 6902) and JSON Pointer (RFC 6901)."""
+
+import copy
+import json
+from typing import Any, Dict, List, Optional
+
+import jsonpatch
+import jsonpointer
+
+
+def format_json(data: Any, stable: bool = False) -> str:
+    """Serialize to json."""
+    return json.dumps(data, indent=4, ensure_ascii=False, sort_keys=not stable) + "\n"
+
+
+def apply_json_fragment(
+        old: Dict[str, Any],
+        new_fragment: Dict[str, Any],
+        acl: List[str],
+) -> Dict[str, Any]:
+    """
+    Replace parts of the old document with 'new_fragment' using ACL restrictions.
+    """
+    full_new_config = copy.deepcopy(old)
+    for acl_item in acl:
+        pointer = jsonpointer.JsonPointer(acl_item)
+
+        try:
+            new_value = pointer.get(new_fragment)
+        except jsonpointer.JsonPointerException:
+            # no value found in new_fragment by the pointer, skip the ACL item
+            continue
+
+        _ensure_pointer_exists(full_new_config, pointer)
+        pointer.set(full_new_config, new_value)
+
+    return full_new_config
+
+
+def _ensure_pointer_exists(doc: Dict[str, Any], pointer: jsonpointer.JsonPointer) -> None:
+    """
+    Ensure that document has all pointer parts (if possible).
+
+    This is workaround for errors of type:
+
+    ```
+    jsonpointer.JsonPointerException: member 'MY_PART' not found in {}
+    ```
+
+    See for details: https://github.com/stefankoegl/python-json-pointer/issues/41
+    """
+    parts_except_the_last = pointer.get_parts()[:-1]
+    doc_pointer: Dict[str, Any] = doc
+    for part in parts_except_the_last:
+        if part not in doc_pointer:
+            # create an empty object by the pointer part
+            doc_pointer[part] = {}
+
+        if isinstance(doc_pointer, dict):
+            # follow the pointer to delve deeper
+            doc_pointer = doc_pointer[part]
+        else:
+            # not a dict - cannot delve deeper
+            break
+
+
+def make_patch(old: Dict[str, Any], new: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Generate a JSON patch by comparing the old document with the new one."""
+    return jsonpatch.make_patch(old, new).patch
+
+
+def apply_patch(content: Optional[bytes], patch_bytes: bytes) -> bytes:
+    """
+    Apply JSON patch to file contents.
+
+    If content is None it is considered that the file does not exist.
+    """
+    old_doc: Any
+    if content is not None:
+        old_doc = json.loads(content)
+    else:
+        old_doc = None
+
+    patch_data = json.loads(patch_bytes)
+    patch = jsonpatch.JsonPatch(patch_data)
+    new_doc = patch.apply(old_doc)
+
+    new_contents = format_json(new_doc, stable=True).encode()
+    return new_contents