annet 0.0__py3-none-any.whl

annet/rulebook/__init__.py

@@ -0,0 +1,114 @@
+import re
+from abc import ABC
+from os import path
+from typing import Iterable, Union
+
+from annet.annlib.lib import mako_render
+from annet.annlib.rbparser.ordering import compile_ordering_text
+from annet.annlib.rbparser.platform import VENDOR_REVERSES
+
+from annet.connectors import CachedConnector
+from annet.rulebook.deploying import compile_deploying_text
+from annet.rulebook.patching import compile_patching_text
+
+
+class RulebookProvider(ABC):
+    def get_rulebook(self, hw):
+        raise NotImplementedError
+
+    def get_root_modules(self) -> Iterable[str]:
+        raise NotImplementedError
+
+
+class _RulebookProviderConnector(CachedConnector[RulebookProvider]):
+    name = "Rulebook provider"
+    ep_name = "rulebook"
+
+
+rulebook_provider_connector = _RulebookProviderConnector()
+
+
+def get_rulebook(hw):
+    return rulebook_provider_connector.get().get_rulebook(hw)
+
+
+class DefaultRulebookProvider(RulebookProvider):
+    root_dir = (path.dirname(__file__),)
+    root_modules = ("annet.rulebook",)
+
+    def __init__(self, root_dir: Union[str, Iterable[str], None] = None,
+                 root_modules: Union[str, Iterable[str], None] = None):
+        self._rulebook_cache = {}
+        self._render_rul_cache = {}
+        self._escaped_rul_cache = {}
+
+        if root_dir is None:
+            pass
+        elif isinstance(root_dir, str):
+            self.root_dir = (root_dir,)
+        else:
+            self.root_dir = tuple(root_dir)
+
+        if root_modules is None:
+            pass
+        elif isinstance(root_modules, str):
+            self.root_modules = (root_modules,)
+        else:
+            self.root_modules = tuple(root_modules)
+
+    def get_root_modules(self):
+        return self.root_modules
+
+    def get_rulebook(self, hw):
+        if hw in self._rulebook_cache:
+            return self._rulebook_cache[hw]
+
+        assert hw.vendor in VENDOR_REVERSES, "Unknown vendor: %s" % (hw.vendor)
+        patching = compile_patching_text(self._render_rul(hw.vendor + ".rul", hw), hw.vendor)
+
+        try:
+            ordering_text = self._render_rul(hw.vendor + ".order", hw)
+        except FileNotFoundError:
+            ordering_text = ""
+        ordering = compile_ordering_text(ordering_text, hw.vendor)
+
+        try:
+            deploying_text = self._render_rul(hw.vendor + ".deploy", hw)
+        except FileNotFoundError:
+            deploying_text = ""
+
+        deploying = compile_deploying_text(deploying_text, hw.vendor)
+
+        self._rulebook_cache[hw] = {
+            "patching": patching,
+            "ordering": ordering,
+            "deploying": deploying,
+        }
+        return self._rulebook_cache[hw]
+
+    def _render_rul(self, name, hw):
+        key = (name, hw)
+        if key not in self._render_rul_cache:
+            self._render_rul_cache[key] = mako_render(self._read_escaped_rul(name), hw=hw)
+        return self._render_rul_cache[key]
+
+    def _read_escaped_rul(self, name):
+        if name in self._escaped_rul_cache:
+            return self._escaped_rul_cache[name]
+        for root_dir in self.root_dir:
+            try:
+                with open(path.join(root_dir, "texts", name), "r") as f:
+                    self._escaped_rul_cache[name] = self._escape_mako(f.read())
+                    return self._escaped_rul_cache[name]
+            except FileNotFoundError:
+                pass
+
+        raise FileNotFoundError(f"Unable to find rul: {name}")
+
+    @staticmethod
+    def _escape_mako(text):
+        # Экранирование всего, что начинается на %, например %comment -> %%comment, чтобы он не интерпретировался
+        # как mako-оператор
+        text = re.sub(r"(?:^|\n)%((?!if\s*|elif\s*|else\s*|endif\s*|for\s*|endfor\s*))", "\n%%\\1", text)
+        text = re.sub(r"(?:^|\n)\s*#.*", "", text)
+        return text

annet/rulebook/arista/iface.py

@@ -0,0 +1,16 @@
+import re
+
+from annet.rulebook import common
+
+
+# Добавление возможности удаления агрегатов, лупбэков, SVI и сабинтерфейсов
+def permanent(rule, key, diff, **kwargs):  # pylint: disable=redefined-outer-name
+    ifname = key[0]
+    # Match group examples:
+    # Group 01: Port-Channel10, Loopback1, Vlan800
+    # Group 02: Ethernet2/1.20, Port-Channel10.200
+    if re.match(r"((?:Port-Channel|Loopback|Vlan)\d+$)|((?:Ethernet|Port-Channel)[\d/]+\.\d+$)", ifname):
+        # Эти интерфейсы можно удалять
+        yield from common.default(rule, key, diff, **kwargs)
+    else:
+        yield from common.permanent(rule, key, diff, **kwargs)

annet/rulebook/aruba/__init__.py

@@ -0,0 +1,16 @@
+from annet.annlib.rulebook import common
+from annet.annlib.types import Op
+
+
+def default_diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
+    diff = common.base_diff(old, new, diff_pre, _pops, moved_to_affected=True)
+    diff[:] = _skip_non_ap_env_affected(diff)
+    return diff
+
+
+def _skip_non_ap_env_affected(diff):
+    for x in diff:
+        if x.op == Op.AFFECTED and not x.children:
+            if x.diff_pre["attrs"]["context"].get("block") != "ap-env":
+                continue
+        yield x

annet/rulebook/aruba/ap_env.py

@@ -0,0 +1,146 @@
+# pylint: disable=unused-argument
+
+from annet.annlib.types import Op
+
+
+try:
+    from annet.executor import CommandList, Command
+except ImportError:
+    from noc.annushka.annet.executor import CommandList, Command
+
+
+def apply(hw, do_commit, do_finalize, **_):
+    before, after = CommandList(), CommandList()
+    if do_commit:
+        after.add_cmd(Command("write memory"))
+    return (before, after)
+
+
+def patch_flag(rule, key, diff, **_):
+    direct, cmd = None, ""
+    if diff[Op.ADDED]:
+        row, _ = diff[Op.ADDED][0]["row"].split(":")
+        cmd = row.replace("_", "-")
+        direct = True
+    elif diff[Op.REMOVED]:
+        row, _ = diff[Op.REMOVED][0]["row"].split(":")
+        cmd = "no " + row.replace("_", "-")
+        direct = False
+    if cmd:
+        yield direct, cmd, None
+
+
+def hostname(rule, key, diff, **_):
+    if diff[Op.ADDED]:
+        yield True, "hostname %s" % key, None
+
+
+def mgmt(rule, key, diff, rule_pre, **_):
+    if not diff[Op.ADDED] and not diff[Op.REMOVED]:
+        return
+    pre_items = rule_pre["items"]
+    unchanged = {k[0]: v[Op.UNCHANGED][0]["row"].split(":")[1] for k, v in pre_items.items() if v[Op.UNCHANGED]}
+    added = {k[0]: v[Op.ADDED][0]["row"].split(":")[1] for k, v in pre_items.items() if v[Op.ADDED]}
+    params = {
+        "ipaddr": None,
+        "netmask": None,
+        "gatewayip": None,
+        "dnsip": None,
+        "domainname": None,
+    }
+    params.update({k: v for k, v in unchanged.items() if k in params})
+    params.update({k: v for k, v in added.items() if k in params})
+    empty = {k: v for k, v in params.items() if v is None}
+    if empty:
+        raise RuntimeError("Failed to determine params %s" % ",".join(empty.keys()))
+    yield True, f"ip-address {params['ipaddr']} {params['netmask']} {params['gatewayip']} {params['dnsip']} {params['domainname']}", None
+
+
+def swarm_mode(rule, key, diff, **_):
+    if diff[Op.ADDED]:
+        row = diff[Op.ADDED][0]["row"]
+        mode = row.split("_")[0]
+        yield True, "swarm-mode %s" % mode, None
+    elif diff[Op.REMOVED]:
+        yield True, "swarm-mode cluster", None
+
+
+def iap_zone(rule, key, diff, **_):
+    if diff[Op.ADDED]:
+        yield True, "zone %s" % key, None
+
+
+def dot11_radio(rule, key, diff, **_):
+    direct, cmd = None, ""
+    if diff[Op.ADDED]:
+        direct, cmd = True, diff[Op.ADDED][0]["row"]
+    elif diff[Op.REMOVED]:
+        direct, cmd = False, "no " + diff[Op.REMOVED][0]["row"]
+    if cmd:
+        cmd = cmd.replace("_", "-")
+        cmd = cmd.replace(":", "-")
+        yield direct, cmd, None
+
+
+def installation_type(rule, key, diff, **_):
+    if diff[Op.ADDED]:
+        row = diff[Op.ADDED][0]["row"]
+        _, installation_place = row.split(":")
+        yield True, "ap-installation %s" % installation_place, None
+    elif diff[Op.REMOVED]:
+        yield True, "ap-installation default", None
+
+
+def wifi_arm(rule, key, diff, root_pre, **_):
+    if key[0].startswith("wifi0"):
+        prefix, cmd = "wifi0", "a-channel"
+    elif key[0].startswith("wifi1"):
+        prefix, cmd = "wifi1", "g-channel"
+    else:
+        raise ValueError("Unknown wifi channel key %r" % key)
+    pre_items = list(root_pre.values())[0]["items"]
+    unchanged = {k[0]: v[Op.UNCHANGED][0]["row"] for k, v in pre_items.items() if v[Op.UNCHANGED]}
+    added = {k[0]: v[Op.ADDED][0]["row"] for k, v in pre_items.items() if v[Op.ADDED]}
+    key_arm_channel = prefix + "_arm_channel"
+    key_arm_power = prefix + "_arm_power_10x"
+    arm_channel, arm_power = "0", "0"
+    for params in [unchanged, added]:
+        if key_arm_channel in params:
+            _, arm_channel = params[key_arm_channel].split(":")
+        if key_arm_power in params:
+            _, arm_power = params[key_arm_power].split(":")
+    yield True, f"{cmd} {arm_channel} {arm_power}", None
+
+
+def ant_gain(rule, key, diff, root_pre, **_):
+    row, value, direct = "", "", None
+    if diff[Op.ADDED]:
+        row = diff[Op.ADDED][0]["row"]
+        _, value = row.split(":")
+        direct = True
+    elif diff[Op.REMOVED]:
+        row = diff[Op.REMOVED][0]["row"]
+        value = "0"
+        direct = False
+    if row:
+        if row.startswith("a_"):
+            cmd = "a-external-antenna"
+        elif row.startswith("g_"):
+            cmd = "g-external-antenna"
+        else:
+            raise ValueError("Unknown row '%s'" % row)
+        yield direct, f"{cmd} {value}", None
+
+
+def ant_pol(rule, key, diff, root_pre, **_):
+    row, value, direct = "", "", None
+    if diff[Op.ADDED]:
+        row, value = diff[Op.ADDED][0]["row"].split(":")
+        direct = True
+    elif diff[Op.REMOVED]:
+        row, _ = diff[Op.REMOVED][0]["row"].split(":")
+        value = "0"
+        direct = False
+    if row:
+        cmd = row.replace("_", "-")
+        yield direct, f"{cmd} {value}", None

annet/rulebook/aruba/misc.py

@@ -0,0 +1,8 @@
+from annet.annlib.rulebook import common
+from annet.annlib.types import Op
+
+
+def syslog_level(rule, key, diff, **_):
+    # syslog-level can be overwritten only
+    if diff[Op.ADDED]:
+        yield from common.default(rule, key, diff)

annet/rulebook/cisco/iface.py

@@ -0,0 +1,68 @@
+from annet.annlib.types import Op
+
+from annet.rulebook import common
+
+
+def diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
+    for iface_row in old:
+        _filter_channel_members(old[iface_row])
+    for iface_row in new:
+        _filter_channel_members(new[iface_row])
+
+    ret = common.default_diff(old, new, diff_pre, _pops)
+    vpn_changed = False
+    for (op, cmd, _, _) in ret:
+        if op in {Op.ADDED, Op.REMOVED}:
+            vpn_changed |= is_vpn_cmd(cmd)
+    if vpn_changed:
+        for cmd in list(old.keys()):
+            if is_ip_cmd(cmd) and not is_vpn_cmd(cmd):
+                del old[cmd]
+        ret = common.default_diff(old, new, diff_pre, _pops)
+    return ret
+
+
+def is_vpn_cmd(cmd):
+    return cmd.startswith("vrf member")
+
+
+def is_ip_cmd(cmd):
+    return cmd.startswith(("ip ", "ipv6 "))
+
+# ===
+
+# Вырезает все команды не разрешенные
+# на членах агрегата. В running-config
+# листинге они наследуются от самого port-channel
+
+
+def _filter_channel_members(tree):
+    if any(is_in_channel(x) for x in tree):
+        for cmd in list(tree.keys()):
+            if not _is_allowed_on_channel(cmd):
+                del tree[cmd]
+
+
+def is_in_channel(cmd_line):
+    """
+    Признак того, что это lagg member
+    """
+    return cmd_line.startswith("channel-group")
+
+
+# Возможно тут есть еще какие-то команды
+def _is_allowed_on_channel(cmd_line):
+    return cmd_line.startswith((
+        "channel-group",
+        "cdp",
+        "description",
+        "inherit",
+        "ip port",
+        "ipv6 port",
+        "mac port",
+        "lacp",
+        "switchport host",
+        "shutdown",
+        "rate-limit cpu",
+        "snmp trap link-status",
+    ))

annet/rulebook/cisco/misc.py

@@ -0,0 +1,57 @@
+import re
+
+from annet.annlib.types import Op
+
+from annet.rulebook import common
+
+
+def ssh_key(rule, key, diff, hw, **_):
+    """
+    При включении ssh надо еще сгенерировать ключ. По конфигу никак не понять есть ли ключ на свитче или нет.
+    """
+    if diff[Op.ADDED]:
+        added = sorted([x["row"] for x in diff[Op.ADDED]])
+        if added == ["ip ssh version 2"]:
+            # Отсыпаем mpdaemon-у подсказок для дополнительной команды при наливке
+            comment = rule["comment"]
+            rule["comment"] = ["!!suppress_errors!!", "!!timeout=240!!"]
+            if hw.Cisco.C2960:
+                yield (False, "crypto key generate rsa modulus 2048", None)
+            else:
+                yield (False, "crypto key generate rsa general-keys modulus 2048", None)
+            rule["comment"] = comment
+    yield from common.default(rule, key, diff)
+
+
+def no_ipv6_nd_suppress_ra(rule, key, diff, **_):
+    """
+    При конфигурации ipv6 nd на нексусах нужно добавлять
+    no ipv6 nd suppress-ra
+    иначе RA не будет включен.
+    К сожалению данной команды не видно в running-config.
+    Поэтому подмешиваем ее в патч вместо генератора
+    """
+    if diff[Op.ADDED]:
+        yield (False, "no ipv6 nd suppress-ra", None)
+    yield from common.default(rule, key, diff)
+
+
+def no_ntp_distribute(rule, key, diff, **_):
+    """
+    Для того, чтобы удалить NTP из CFS, сначала нужно сбросить активные
+    NTP сессии.
+    """
+    if diff[Op.REMOVED]:
+        yield (False, "clear ntp session", None)
+    yield from common.default(rule, key, diff)
+
+
+def banner_login(rule, key, diff, **_):
+    if diff[Op.REMOVED]:
+        yield (False, "no banner login", None)
+    elif diff[Op.ADDED]:
+        # Убираем дополнительный экранирующий сиимвол
+        key = re.sub(r"\^C", "^", key[0])
+        yield (False, f"banner login {key}", None)
+    else:
+        yield from common.default(rule, key, diff)

annet/rulebook/cisco/vlandb.py

@@ -0,0 +1,90 @@
+import re
+
+from annet.annlib.lib import cisco_collapse_vlandb as collapse_vlandb
+from annet.annlib.lib import cisco_expand_vlandb as expand_vlandb
+from annet.annlib.types import Op
+
+
+VLANDB_CHUNK = 15
+SWTRUNK_CHUNK = 5
+
+
+# =====
+def simple(rule, key, diff, hw, **_):
+    yield from _process_vlandb(rule, key, diff, hw, False, VLANDB_CHUNK)
+
+
+def swtrunk(rule, key, diff, hw, **_):
+    yield from _process_vlandb(rule, key, diff, hw, True, SWTRUNK_CHUNK)
+
+
+# =====
+def _process_vlandb(rule, key, diff, hw, explicit_changing, multi_chunk):
+    # pylint: disable=unused-argument
+    for affected in diff[Op.AFFECTED]:
+        # Изменилось содержимое блока vlan
+        yield (True, affected["row"], affected["children"])
+
+    (prefix, new, new_blocks) = _parse_vlancfg_actions(diff[Op.ADDED])
+    (prefix2, old, old_blocks) = _parse_vlancfg_actions(diff[Op.REMOVED])
+    if not prefix:
+        prefix = prefix2
+
+    if len(diff[Op.ADDED]) == 1 and len(new) == 0:
+        # switchport trunk allowed vlan none
+        yield (True, "%s none" % prefix, None)
+        return
+    for vlan_id in ((set(old_blocks.keys()) - set(new_blocks)) & new):
+        # Удалено содержимое блока vlan, но сам влан остался
+        yield (True, "%s %s" % (prefix, vlan_id), old_blocks[vlan_id])
+
+    removed = old.difference(new)
+    added = new.difference(old)
+    if hw.Catalyst:
+        # Каталисты не перечисляют вланы в batch режиме, если они представлены как блоки
+        added -= new_blocks.keys()
+
+    if removed:
+        collapsed = collapse_vlandb(removed, hw.Catalyst)
+        for chunk in _chunked(collapsed, multi_chunk):
+            yield (False, "no %s%s%s" % (prefix, " remove " if explicit_changing else " ", ",".join(chunk)), None)
+
+    if added:
+        collapsed = collapse_vlandb(added, hw.Catalyst)
+        for chunk in _chunked(collapsed, multi_chunk):
+            yield (True, "%s%s%s" % (prefix, " add " if explicit_changing else " ", ",".join(chunk)), None)
+    if new_blocks:
+        for vlan_id, block in new_blocks.items():
+            yield (True, "%s %s" % (prefix, vlan_id), block)
+
+
+def _chunked(items, size):
+    for offset in range(0, len(items), size):
+        yield items[offset:offset + size]
+
+
+def _parse_vlancfg_actions(actions):
+    prefix = None
+    vlandb = set()
+    blocks = {}
+    for action in actions:
+        (prefix, part) = _parse_vlancfg(action["row"])
+        if action["children"]:
+            assert len(part) == 1, "vlandb block must contain one and only one vlanid: %s" % action["row"]
+            blocks[list(part)[0]] = action["children"]
+        vlandb.update(part)
+    return (prefix, vlandb, blocks)
+
+
+def _parse_vlancfg(row):
+    # иногда циски ставят пробелы между влан ренджами, а иногда нет.
+    words = re.sub(r",\s+", ",", row).split()
+
+    if words[-1] == "none":
+        # switchport trunk allowed vlan none
+        return (" ".join(words[:-1]), set())
+    assert re.match(r"[\d,-]+$", words[-1]), "Unable to parse vlancfg row: %s" % row
+    prefix = " ".join(words[:-2] if words[-2] == "add" else words[:-1])
+    vlancfg = words[-1]
+    vlandb = expand_vlandb(vlancfg)
+    return (prefix, vlandb)

annet/rulebook/common.py

@@ -0,0 +1,19 @@
+import functools
+import importlib
+
+from annet.annlib.rulebook import common  # pylint: disable=unused-import # noqa: F401,F403
+from annet.annlib.rulebook.common import *  # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403
+
+
+@functools.lru_cache()
+def import_rulebook_function(name):
+    from . import rulebook_provider_connector
+
+    index = name.rindex(".")
+    for root in rulebook_provider_connector.get().get_root_modules():
+        try:
+            module = importlib.import_module(f"{root}.{name[:index]}", package=__name__.rsplit(".", 1)[0])
+            return getattr(module, name[index + 1:])
+        except ImportError:
+            pass
+    raise ImportError(f"Could not import {name}")

annet/rulebook/deploying.py

@@ -0,0 +1,87 @@
+import functools
+from collections import OrderedDict as odict
+from collections import namedtuple
+
+from annet.annlib.rbparser import platform, syntax
+from annet.annlib.rbparser.deploying import compile_messages
+from valkit.common import valid_bool, valid_number, valid_string_list
+from valkit.python import valid_object_path
+
+from .common import import_rulebook_function
+
+
+Answer = namedtuple("Answer", ("text", "send_nl"))
+DEFAULT_TIMEOUT = 30
+DEFAULT_SEND_NL = True
+DEFAULT_APPLY_LOGIC = "common.apply"
+
+
+# =====
+@functools.lru_cache()
+def compile_deploying_text(text, vendor):
+    return _compile_deploying(
+        tree=syntax.parse_text(text, params_scheme={
+            "timeout": {
+                "validator": lambda arg: valid_number(arg, min=1, type=float),
+                "default": 30,
+            },
+            "send_nl": {
+                "validator": valid_bool,
+                "default": True,
+            },
+            "apply_logic": {
+                "validator": valid_object_path,
+                "default": DEFAULT_APPLY_LOGIC,
+            },
+            "ifcontext": {
+                "validator": valid_string_list,
+                "default": [],
+            }
+        }),
+        reverse_prefix=platform.VENDOR_REVERSES[vendor],
+    )
+
+
+# =====
+def _compile_deploying(tree, reverse_prefix):
+    deploying = odict()
+    for (rule_id, attrs) in tree.items():
+        if attrs["type"] == "normal" and not attrs["row"].startswith(("ignore:", "dialog:")):
+            (ignore, dialogs) = compile_messages(attrs["children"])
+            deploying[rule_id] = {
+                "attrs": {
+                    "regexp": syntax.compile_row_regexp(attrs["row"]),
+                    "timeout": attrs["params"]["timeout"],
+                    "apply_logic": import_rulebook_function(attrs["params"]["apply_logic"]),
+                    "ignore": ignore,
+                    "dialogs": dialogs,
+                    "ifcontext": attrs["params"]["ifcontext"],
+                },
+                "children": _compile_deploying(attrs["children"], reverse_prefix),
+            }
+    return deploying
+
+
+def match_deploy_rule(rules, cmd_path, context):
+    for (depth, row) in enumerate(cmd_path):
+        for rule in rules.values():
+            if rule["attrs"]["regexp"].match(row):
+                ifcontext = rule["attrs"]["ifcontext"]
+                if syntax.match_context(ifcontext, context):
+                    if depth == len(cmd_path) - 1:
+                        return rule
+                    else:
+                        rules = rule["children"]
+                        if len(rules) == 0:
+                            break
+    # default match
+    return {
+        "attrs": {
+            "regexp": syntax.compile_row_regexp("~"),
+            "timeout": DEFAULT_TIMEOUT,
+            "apply_logic": import_rulebook_function(DEFAULT_APPLY_LOGIC),
+            "ignore": [],
+            "dialogs": odict(),
+        },
+        "children": odict(),
+    }