annet 1.0.4__tar.gz → 1.1.1__tar.gz

{annet-1.0.4/annet.egg-info → annet-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: annet
-Version: 1.0.4
+Version: 1.1.1
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -15,7 +15,6 @@ Requires-Dist: PyYAML>=6.0.1
 Requires-Dist: Pygments>=2.14.0
 Requires-Dist: Mako>=1.2.4
 Requires-Dist: Jinja2>=3.1.2
-Requires-Dist: psutil>=5.8.0
 Requires-Dist: packaging>=23.2
 Requires-Dist: contextlog>=1.1
 Requires-Dist: valkit>=0.1.4
@@ -23,7 +22,7 @@ Requires-Dist: yarl>=1.8.2
 Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.2.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.2.1; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: provides-extra

{annet-1.0.4 → annet-1.1.1}/annet/adapters/netbox/common/query.py

@@ -5,7 +5,7 @@ from typing import cast, List, Union, Iterable, Optional, TypedDict
 from annet.storage import Query
 
 FIELD_VALUE_SEPARATOR = ":"
-ALLOWED_GLOB_GROUPS = ["site", "tag", "role", "device_type"]
+ALLOWED_GLOB_GROUPS = ["site", "tag", "role", "device_type", "status", "tenant"]
 
 
 class Filter(TypedDict, total=False):
@@ -14,6 +14,8 @@ class Filter(TypedDict, total=False):
     role: list[str]
     name: list[str]
     device_type: list[str]
+    status: list[str]
+    tenant: list[str]
 
 
 @dataclass

{annet-1.0.4 → annet-1.1.1}/annet/annlib/patching.py

@@ -2,7 +2,7 @@ import copy
 import operator
 import textwrap
 from collections import OrderedDict as odict
-from typing import (  # pylint: disable=unused-import
+from typing import (
     Any,
     Dict,
     Iterator,
@@ -177,7 +177,7 @@ class Orderer:
     def rule_weight(self, row, rule, regexp_key):
         return len(set(row).intersection(set(rule["attrs"][regexp_key].pattern))) / len(row)
 
-    def get_order(self, row, cmd_direct):
+    def get_order(self, row, cmd_direct, scope: str | None = None):
         f_order = None
         f_weight = 0
         f_rule = ""
@@ -186,6 +186,15 @@ class Orderer:
         block_exit = platform.VENDOR_EXIT[self.vendor]
 
         for (order, (raw_rule, rule)) in enumerate(ordering.items()):
+            if (
+                (rule_scope := rule["attrs"]["scope"]) is not None
+                and scope not in rule_scope
+            ):
+                continue
+
+            if rule["attrs"]["global"]:
+                children.append((raw_rule, rule))
+
             direct_matched = bool(rule["attrs"]["direct_regexp"].match(row))
             if not rule["attrs"]["order_reverse"] and (direct_matched or rule["attrs"]["reverse_regexp"].match(row)):
                 # если не указано order_reverse - правило считается прямым
@@ -395,7 +404,7 @@ def make_patch(pre, rb, hw, add_comments, orderer=None, _root_pre=None, do_commi
         for (key, diff) in content["items"].items():
             # чтобы logic не мог поменять атрибуты
             rule_pre = content.copy()
-            attrs = rule_pre["attrs"].copy()
+            attrs = copy.deepcopy(rule_pre["attrs"])
 
             iterable = attrs["logic"](
                 rule=attrs,
@@ -416,7 +425,7 @@ def make_patch(pre, rb, hw, add_comments, orderer=None, _root_pre=None, do_commi
                             patch_row = "%s %s" % (row, comments)
 
                     # pylint: disable=unused-variable
-                    (order, order_direct, ordering, order_rule) = orderer.get_order(row, direct)
+                    (order, order_direct, ordering, order_rule) = orderer.get_order(row, direct, scope="patch")
                     fmt_row = patch_row
                     # fmt_row += "  # %s" % str(order_rule)  # uncomment to debug ordering
 
@@ -544,8 +553,7 @@ def _select_match(matches, rules):
         for (rule, is_cr_allowed) in map(operator.itemgetter(0), matches):
             if is_cr_allowed:
                 local_children = merge_dicts(local_children, rule["children"]["local"])
-            # optional break on is_cr_allowed==False?
-
+                # optional break on is_cr_allowed==False?
                 global_children = merge_dicts(global_children, rule["children"]["global"])
 
     global_children = merge_dicts(global_children, rules["global"])
@@ -555,9 +563,10 @@ def _select_match(matches, rules):
         "global": global_children,
     }
 
-    match = {"attrs": f_rule["attrs"]}
+    match = {"attrs": copy.deepcopy(f_rule["attrs"])}
     match.update(f_other)
-    return (match, children_rules)
+
+    return match, children_rules
 
 
 def _rules_local_global(rules):

{annet-1.0.4 → annet-1.1.1}/annet/annlib/rbparser/ordering.py

@@ -2,7 +2,7 @@ import functools
 import re
 from collections import OrderedDict as odict
 
-from valkit.common import valid_bool
+from valkit.common import valid_bool, valid_string_list
 
 from . import platform, syntax
 
@@ -16,6 +16,14 @@ def compile_ordering_text(text, vendor):
                 "validator": valid_bool,
                 "default":   False,
             },
+            "global": {
+                "validator": valid_bool,
+                "default": False,
+            },
+            "scope": {
+                "validator": valid_string_list,
+                "default": None,
+            }
         }),
         reverse_prefix=platform.VENDOR_REVERSES[vendor],
     )
@@ -44,6 +52,8 @@ def _compile_ordering(tree, reverse_prefix):
                         syntax.compile_row_regexp(re.sub(r"^%s\s+" % (reverse_prefix), "", attrs["row"]))
                     ),
                     "order_reverse": attrs["params"]["order_reverse"],
+                    "global": attrs["params"]["global"],
+                    "scope": attrs["params"]["scope"],
                     "raw_rule": attrs["raw_rule"],
                     "context": attrs["context"],
                 },

{annet-1.0.4 → annet-1.1.1}/annet/annlib/rbparser/platform.py

@@ -25,7 +25,7 @@ VENDOR_DIFF = {
     "routeros": "common.default_diff",
     "aruba": "aruba.default_diff",
     "pc": "common.default_diff",
-    "ribbon": "ribbon.default_diff",
+    "ribbon": "common.default_diff",
     "b4com": "common.default_diff",
 }
 
@@ -40,7 +40,7 @@ VENDOR_DIFF_ORDERED = {
     "routeros": "common.ordered_diff",
     "aruba": "common.ordered_diff",
     "pc": "common.ordered_diff",
-    "ribbon": "ribbon.default_diff",
+    "ribbon": "common.ordered_diff",
     "b4com": "common.ordered_diff",
 }
 

{annet-1.0.4 → annet-1.1.1}/annet/annlib/tabparser.py

@@ -1,6 +1,8 @@
 import dataclasses
 import itertools
+import json
 import re
+import textwrap
 from collections import OrderedDict as odict
 from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union, List
 
@@ -70,10 +72,10 @@ class CommonFormatter:
         self._block_end = ""
         self._statement_end = ""
 
-    def split(self, text):
+    def split(self, text: str):
         return list(filter(None, text.split("\n")))
 
-    def join(self, config):
+    def join(self, config: "PatchTree"):
         return "\n".join(
             _filtered_block_marks(
                 self._indent_blocks(self._blocks(config, is_patch=False))
@@ -86,14 +88,14 @@ class CommonFormatter:
     def diff(self, diff):
         return list(self.diff_generator(diff))
 
-    def patch(self, patch):
+    def patch(self, patch: "PatchTree") -> str:
         return "\n".join(
             _filtered_block_marks(
                 self._indent_blocks(self._blocks(patch, is_patch=True))
             )
         )
 
-    def cmd_paths(self, patch):
+    def cmd_paths(self, patch: "PatchTree") -> odict:
         ret = odict()
         path = []
         for row, context in self.blocks_and_context(patch, is_patch=True):
@@ -175,7 +177,7 @@ class CommonFormatter:
                 )
                 yield BlockEnd, None
 
-    def _blocks(self, tree, is_patch):
+    def _blocks(self, tree: "PatchTree", is_patch: bool):
         for row, _context in self.blocks_and_context(tree, is_patch):
             yield row
 
@@ -386,7 +388,32 @@ class AsrFormatter(BlockExitFormatter):
 
 
 class JuniperFormatter(CommonFormatter):
-    patch_set_prefix = "set "
+    patch_set_prefix = "set"
+
+    @dataclasses.dataclass
+    class Comment:
+        begin = "/*"
+        end = "*/"
+
+        row: str
+        comment: str
+
+        def __post_init__(self):
+            self.row = self.row.strip()
+            self.comment = self.comment.strip()
+
+        @classmethod
+        def loads(cls, value: str):
+            return cls(
+                **json.loads(
+                    value.removeprefix(cls.begin)
+                    .removesuffix(cls.end)
+                    .strip()
+                )
+            )
+
+        def dumps(self):
+            return json.dumps({"row": self.row, "comment": self.comment})
 
     def __init__(self, indent="    "):
         super().__init__(indent)
@@ -395,20 +422,32 @@ class JuniperFormatter(CommonFormatter):
         self._statement_end = ";"
         self._endofline_comment = "; ##"
 
-    def split(self, text):
-        sub_regexs = (
+        self._sub_regexs = (
             (re.compile(self._block_begin + r"\s*" + self._block_end + r"$"), ""),  # collapse empty blocks
             (re.compile(self._block_begin + "(\t# .+)?$"), ""),
             (re.compile(self._statement_end + r"$"), ""),
             (re.compile(r"\s*" + self._block_end + "(\t# .+)?$"), ""),
             (re.compile(self._endofline_comment + r".*$"), ""),
         )
-        split = []
-        for line in text.split("\n"):
-            for (regex, repl_line) in sub_regexs:
-                line = regex.sub(repl_line, line)
-            split.append(line)
-        return list(filter(None, split))
+
+    def sub_regexs(self, value: str) -> str:
+        for (regex, repl_line) in self._sub_regexs:
+            value = regex.sub(repl_line, value)
+        return value
+
+    def split(self, text: str) -> list[str]:
+        comment_begin, comment_end = map(re.escape, (self.Comment.begin, self.Comment.end))
+        comment_regexp = re.compile(fr"(\s+{comment_begin})((?:(?!{comment_end}).)*)({comment_end})")
+
+        result = []
+        lines = text.split("\n")
+        for i, line in enumerate(lines):
+            line = self.sub_regexs(line)
+            if i + 1 < len(lines) and (m := comment_regexp.match(line)):
+                line = f"{m.group(1)} {self.Comment(self.sub_regexs(lines[i + 1]), m.group(2)).dumps()} {m.group(3)}"
+            result.append(line)
+
+        return list(filter(None, result))
 
     def join(self, config):
         return "\n".join(_filtered_block_marks(self._formatted_blocks(self._indented_blocks(config))))
@@ -419,6 +458,13 @@ class JuniperFormatter(CommonFormatter):
     def patch_plain(self, patch):
         return list(self.cmd_paths(patch).keys())
 
+    def _blocks(self, tree: "PatchTree", is_patch: bool):
+        for row in super()._blocks(tree, is_patch):
+            if isinstance(row, str) and row.startswith(self.Comment.begin):
+                yield f"{self.Comment.begin} {self.Comment.loads(row).comment} {self.Comment.end}"
+            else:
+                yield row
+
     def _formatted_blocks(self, blocks):
         level = 0
         line = None
@@ -430,33 +476,48 @@ class JuniperFormatter(CommonFormatter):
             elif new_line is BlockEnd:
                 level -= 1
                 if isinstance(line, str):
-                    yield line + self._statement_end
+                    yield line + ("" if line.endswith(self.Comment.end) else self._statement_end)
                 yield self._indent * level + self._block_end
             elif isinstance(line, str):
-                yield line + self._statement_end
+                yield line + ("" if line.endswith(self.Comment.end) else self._statement_end)
             line = new_line
         if isinstance(line, str):
             yield line + self._statement_end
 
-    def cmd_paths(self, patch, _prev=""):
+    def cmd_paths(self, patch, _prev=tuple()):
         commands = odict()
         for item in patch.itms:
             key, childs, context = item.row, item.child, item.context
+
             if childs:
-                for k, v in self.cmd_paths(childs, _prev + " " + key).items():
+                for k, v in self.cmd_paths(childs, (*_prev, key.strip())).items():
                     commands[k] = v
             else:
-                if key.startswith("delete"):
-                    cmd = "delete" + _prev + " " + key.replace("delete", "", 1).strip()
+                if "comment" in context:
+                    value = (
+                        ""
+                        if key.startswith("delete")
+                        else context["comment"]
+                    )
+
+                    cmd = "\n".join(
+                        (
+                            "edit " + " ".join(_prev),
+                            " ".join(("annotate", context["row"].split(" ")[0], f'"{value}"')),
+                            "exit"
+                        )
+                    )
+                elif key.startswith("delete"):
+                    cmd = " ".join(("delete", *_prev, key.replace("delete", "", 1).strip()))
                 elif key.startswith("activate"):
-                    cmd = "activate" + _prev + " " + key.replace("activate", "", 1).strip()
+                    cmd = " ".join(("activate", *_prev, key.replace("activate", "", 1).strip()))
                 elif key.startswith("deactivate"):
-                    cmd = "deactivate" + _prev + " " + key.replace("deactivate", "", 1).strip()
+                    cmd = " ".join(("deactivate", *_prev, key.replace("deactivate", "", 1).strip()))
                 else:
-                    cmd = (self.patch_set_prefix + _prev.strip()).strip() + " " + key
+                    cmd = " ".join((self.patch_set_prefix, *_prev, key.strip()))
+
                 # Expanding [ a b c ] junipers list of arguments
-                matches = re.search(r"^(.*)\s+\[(.+)\]$", cmd)
-                if matches:
+                if matches := re.search(r"^(.*)\s+\[(.+)\]$", cmd):
                     for c in matches.group(2).split(" "):
                         if c.strip():
                             cmd = " ".join([matches.group(1), c])
@@ -490,7 +551,7 @@ class JuniperList:
 
 
 class NokiaFormatter(JuniperFormatter):
-    patch_set_prefix = "/configure "
+    patch_set_prefix = "/configure"
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -517,18 +578,18 @@ class NokiaFormatter(JuniperFormatter):
         finish = finish if finish is not None else len(ret)
         return ret[start:finish]
 
-    def cmd_paths(self, patch, _prev=""):
+    def cmd_paths(self, patch, _prev=tuple()):
         commands = odict()
         for item in patch.itms:
             key, childs, context = item.row, item.child, item.context
             if childs:
-                for k, v in self.cmd_paths(childs, _prev + " " + key).items():
+                for k, v in self.cmd_paths(childs, (*_prev, key.strip())).items():
                     commands[k] = v
             else:
                 if key.startswith("delete"):
-                    cmd = "/configure delete" + _prev + " " + key.replace("delete", "", 1).strip()
+                    cmd = " ".join((self.patch_set_prefix, "delete", *_prev, key.replace("delete", "", 1).strip()))
                 else:
-                    cmd = self.patch_set_prefix + _prev.strip() + " " + key
+                    cmd = " ".join((self.patch_set_prefix, *_prev, key.strip()))
                 # Expanding [ a b c ] junipers list of arguments
                 matches = re.search(r"^(.*)\s+\[(.+)\]$", cmd)
                 if matches:

annet-1.1.1/annet/executor.py

@@ -0,0 +1,172 @@
+import asyncio
+import os
+import statistics
+from abc import ABC, abstractmethod
+from functools import partial
+from operator import itemgetter
+from typing import Any, List, Optional
+
+import colorama
+from annet.annlib.command import Command, CommandList, Question  # noqa: F401
+
+
+class CommandResult(ABC):
+    @abstractmethod
+    def get_out(self) -> str:
+        pass
+
+
+class ExecutorException(Exception):
+    def __init__(self, *args: List[Any], auxiliary: Optional[Any] = None, **kwargs: object):
+        self.auxiliary = auxiliary
+        super().__init__(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        return "%s(args=%r,auxiliary=%s)" % (self.__class__.__name__, self.args, self.auxiliary)
+
+
+class ExecException(ExecutorException):
+    def __init__(self, msg: str, cmd: str, res: str, **kwargs):
+        super().__init__(**kwargs)
+        self.args = msg, cmd, res
+        self.kwargs = kwargs
+        self.msg = msg
+        self.cmd = cmd
+        self.res = res
+
+    def __str__(self) -> str:
+        return str(self.msg)
+
+    def __repr__(self) -> str:
+        return "%s<%s, %s>" % (self.__class__.__name__, self.msg, self.cmd)
+
+
+class BadCommand(ExecException):
+    pass
+
+
+class NonzeroRetcode(ExecException):
+    pass
+
+
+class CommitException(ExecException):
+    pass
+
+
+def _show_type_summary(caption, items, total, stat_items=None):
+    if items:
+        if not stat_items:
+            stat = ""
+        else:
+            avg = statistics.mean(stat_items)
+            stat = "   %(min).1f/%(max).1f/%(avg).1f/%(stdev)s  (min/max/avg/stdev)" % dict(
+                min=min(stat_items),
+                max=max(stat_items),
+                avg=avg,
+                stdev="-" if len(stat_items) < 2 else "%.1f" % statistics.stdev(stat_items, xbar=avg)
+            )
+
+        print("%-8s %d of %d%s" % (caption, len(items), total, stat))
+
+
+def show_bulk_report(hostnames, res, durations, log_dir):
+    total = len(hostnames)
+    if not total:
+        return
+
+    colorama.init()
+
+    print("\n====== bulk deploy report ======")
+
+    done = [host for (host, hres) in res.items() if not isinstance(hres, Exception)]
+    cancelled = [host for (host, hres) in res.items() if isinstance(hres, asyncio.CancelledError)]
+    failed = [host for (host, hres) in res.items() if isinstance(hres, Exception) and host not in cancelled]
+    lost = [host for host in hostnames if host not in res]
+    limit = 30
+
+    _show_type_summary("Done :", done, total, [durations[h] for h in done])
+    _print_limit(done, partial(_print_hostname, style=colorama.Fore.GREEN), limit, total)
+
+    _show_type_summary("Failed :", failed, total, [durations[h] for h in failed])
+
+    _print_limit(failed, partial(_print_failed, res=res), limit, total)
+
+    _show_type_summary("Cancelled :", cancelled, total, [durations[h] for h in cancelled if durations[h] is not None])
+    _print_limit(cancelled, partial(_print_hostname, style=colorama.Fore.RED), limit, total)
+
+    _show_type_summary("Lost :", lost, total)
+    _print_limit(lost, _print_hostname, limit, total)
+
+    err_limit = 5
+    if failed:
+        errs = {}
+        for hostname in failed:
+            fmt_err = _format_exc(res[hostname])
+            if fmt_err in errs:
+                errs[fmt_err] += 1
+            else:
+                errs[fmt_err] = 1
+        print("Top errors :")
+        for fmt_err, n in sorted(errs.items(), key=itemgetter(1), reverse=True)[:err_limit]:
+            print("  %-4d %s" % (n, fmt_err))
+        print("\n", end="")
+
+    if log_dir:
+        print("See deploy logs in %s/\n" % os.path.relpath(log_dir))
+
+
+def _format_exc(exc):
+    if isinstance(exc, ExecException):
+        cmd = str(exc.cmd)
+        if len(cmd) > 50:
+            cmd = cmd[:50] + "~.."
+        return "'%s', cmd '%s'" % (exc.msg, cmd)
+    elif isinstance(exc, ExecutorException):
+        return "%s%r" % (exc.__class__.__name__, exc.args)  # исключить многословный auxiliary
+    else:
+        return repr(exc)
+
+
+def _print_hostname(host, style=None):
+    if style:
+        host = style + host + colorama.Style.RESET_ALL
+    print("  %s" % host)
+
+
+def _print_limit(items, printer, limit, total, end="\n"):
+    if not items:
+        return
+    if len(items) > limit and len(items) > total * 0.7:
+        print("  ... %d hosts" % len(items))
+    for host in items[:limit]:
+        printer(host)
+    if len(items) > limit:
+        print("  ... %d more hosts" % (len(items) - limit))
+
+    print(end, end="")
+
+
+def _print_failed(host, res):
+    exc = res[host]
+    color = colorama.Fore.YELLOW if isinstance(exc, Warning) else colorama.Fore.RED
+    print("  %s - %s" % (color + host + colorama.Style.RESET_ALL, _format_exc(exc)))
+
+
+class DeferredFileWrite:
+    def __init__(self, file, mode="r"):
+        self._file = file
+        wrapper = {"w": "a", "wb": "ab"}
+        if mode in wrapper:
+            self._mode = wrapper[mode]
+        else:
+            raise Exception()
+
+    def write(self, data):
+        with open(self._file, self._mode) as fh:
+            fh.write(data)
+
+    def close(self):
+        pass
+
+    def flush(self):
+        pass