annet 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

annet/annlib/patching.py

@@ -186,6 +186,9 @@ class Orderer:
         block_exit = platform.VENDOR_EXIT[self.vendor]
 
         for (order, (raw_rule, rule)) in enumerate(ordering.items()):
+            if rule["attrs"]["global"]:
+                children.append((raw_rule, rule))
+
             direct_matched = bool(rule["attrs"]["direct_regexp"].match(row))
             if not rule["attrs"]["order_reverse"] and (direct_matched or rule["attrs"]["reverse_regexp"].match(row)):
                 # если не указано order_reverse - правило считается прямым
@@ -395,7 +398,7 @@ def make_patch(pre, rb, hw, add_comments, orderer=None, _root_pre=None, do_commi
         for (key, diff) in content["items"].items():
             # чтобы logic не мог поменять атрибуты
             rule_pre = content.copy()
-            attrs = rule_pre["attrs"].copy()
+            attrs = copy.deepcopy(rule_pre["attrs"])
 
             iterable = attrs["logic"](
                 rule=attrs,
@@ -544,8 +547,7 @@ def _select_match(matches, rules):
         for (rule, is_cr_allowed) in map(operator.itemgetter(0), matches):
             if is_cr_allowed:
                 local_children = merge_dicts(local_children, rule["children"]["local"])
-            # optional break on is_cr_allowed==False?
-
+                # optional break on is_cr_allowed==False?
                 global_children = merge_dicts(global_children, rule["children"]["global"])
 
     global_children = merge_dicts(global_children, rules["global"])
@@ -555,9 +557,10 @@ def _select_match(matches, rules):
         "global": global_children,
     }
 
-    match = {"attrs": f_rule["attrs"]}
+    match = {"attrs": copy.deepcopy(f_rule["attrs"])}
     match.update(f_other)
-    return (match, children_rules)
+
+    return match, children_rules
 
 
 def _rules_local_global(rules):

annet/annlib/rbparser/ordering.py

@@ -16,6 +16,10 @@ def compile_ordering_text(text, vendor):
                 "validator": valid_bool,
                 "default":   False,
             },
+            "global": {
+                "validator": valid_bool,
+                "default": False,
+            }
         }),
         reverse_prefix=platform.VENDOR_REVERSES[vendor],
     )
@@ -44,6 +48,7 @@ def _compile_ordering(tree, reverse_prefix):
                         syntax.compile_row_regexp(re.sub(r"^%s\s+" % (reverse_prefix), "", attrs["row"]))
                     ),
                     "order_reverse": attrs["params"]["order_reverse"],
+                    "global": attrs["params"]["global"],
                     "raw_rule": attrs["raw_rule"],
                     "context": attrs["context"],
                 },

annet/annlib/rbparser/platform.py

@@ -25,7 +25,7 @@ VENDOR_DIFF = {
     "routeros": "common.default_diff",
     "aruba": "aruba.default_diff",
     "pc": "common.default_diff",
-    "ribbon": "ribbon.default_diff",
+    "ribbon": "common.default_diff",
     "b4com": "common.default_diff",
 }
 
@@ -40,7 +40,7 @@ VENDOR_DIFF_ORDERED = {
     "routeros": "common.ordered_diff",
     "aruba": "common.ordered_diff",
     "pc": "common.ordered_diff",
-    "ribbon": "ribbon.default_diff",
+    "ribbon": "common.ordered_diff",
     "b4com": "common.ordered_diff",
 }
 

annet/annlib/tabparser.py

@@ -1,6 +1,8 @@
 import dataclasses
 import itertools
+import json
 import re
+import textwrap
 from collections import OrderedDict as odict
 from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union, List
 
@@ -70,10 +72,10 @@ class CommonFormatter:
         self._block_end = ""
         self._statement_end = ""
 
-    def split(self, text):
+    def split(self, text: str):
         return list(filter(None, text.split("\n")))
 
-    def join(self, config):
+    def join(self, config: "PatchTree"):
         return "\n".join(
             _filtered_block_marks(
                 self._indent_blocks(self._blocks(config, is_patch=False))
@@ -86,14 +88,14 @@ class CommonFormatter:
     def diff(self, diff):
         return list(self.diff_generator(diff))
 
-    def patch(self, patch):
+    def patch(self, patch: "PatchTree") -> str:
         return "\n".join(
             _filtered_block_marks(
                 self._indent_blocks(self._blocks(patch, is_patch=True))
             )
         )
 
-    def cmd_paths(self, patch):
+    def cmd_paths(self, patch: "PatchTree") -> odict:
         ret = odict()
         path = []
         for row, context in self.blocks_and_context(patch, is_patch=True):
@@ -175,7 +177,7 @@ class CommonFormatter:
                 )
                 yield BlockEnd, None
 
-    def _blocks(self, tree, is_patch):
+    def _blocks(self, tree: "PatchTree", is_patch: bool):
         for row, _context in self.blocks_and_context(tree, is_patch):
             yield row
 
@@ -386,7 +388,32 @@ class AsrFormatter(BlockExitFormatter):
 
 
 class JuniperFormatter(CommonFormatter):
-    patch_set_prefix = "set "
+    patch_set_prefix = "set"
+
+    @dataclasses.dataclass
+    class Comment:
+        begin = "/*"
+        end = "*/"
+
+        row: str
+        comment: str
+
+        def __post_init__(self):
+            self.row = self.row.strip()
+            self.comment = self.comment.strip()
+
+        @classmethod
+        def loads(cls, value: str):
+            return cls(
+                **json.loads(
+                    value.removeprefix(cls.begin)
+                    .removesuffix(cls.end)
+                    .strip()
+                )
+            )
+
+        def dumps(self):
+            return json.dumps({"row": self.row, "comment": self.comment})
 
     def __init__(self, indent="    "):
         super().__init__(indent)
@@ -395,20 +422,32 @@ class JuniperFormatter(CommonFormatter):
         self._statement_end = ";"
         self._endofline_comment = "; ##"
 
-    def split(self, text):
-        sub_regexs = (
+        self._sub_regexs = (
             (re.compile(self._block_begin + r"\s*" + self._block_end + r"$"), ""),  # collapse empty blocks
             (re.compile(self._block_begin + "(\t# .+)?$"), ""),
             (re.compile(self._statement_end + r"$"), ""),
             (re.compile(r"\s*" + self._block_end + "(\t# .+)?$"), ""),
             (re.compile(self._endofline_comment + r".*$"), ""),
         )
-        split = []
-        for line in text.split("\n"):
-            for (regex, repl_line) in sub_regexs:
-                line = regex.sub(repl_line, line)
-            split.append(line)
-        return list(filter(None, split))
+
+    def sub_regexs(self, value: str) -> str:
+        for (regex, repl_line) in self._sub_regexs:
+            value = regex.sub(repl_line, value)
+        return value
+
+    def split(self, text: str) -> list[str]:
+        comment_begin, comment_end = map(re.escape, (self.Comment.begin, self.Comment.end))
+        comment_regexp = re.compile(fr"(\s+{comment_begin})((?:(?!{comment_end}).)*)({comment_end})")
+
+        result = []
+        lines = text.split("\n")
+        for i, line in enumerate(lines):
+            line = self.sub_regexs(line)
+            if i + 1 < len(lines) and (m := comment_regexp.match(line)):
+                line = f"{m.group(1)} {self.Comment(self.sub_regexs(lines[i + 1]), m.group(2)).dumps()} {m.group(3)}"
+            result.append(line)
+
+        return list(filter(None, result))
 
     def join(self, config):
         return "\n".join(_filtered_block_marks(self._formatted_blocks(self._indented_blocks(config))))
@@ -433,30 +472,45 @@ class JuniperFormatter(CommonFormatter):
                     yield line + self._statement_end
                 yield self._indent * level + self._block_end
             elif isinstance(line, str):
-                yield line + self._statement_end
+                yield line + ("" if line.endswith(self.Comment.end) else self._statement_end)
             line = new_line
         if isinstance(line, str):
             yield line + self._statement_end
 
-    def cmd_paths(self, patch, _prev=""):
+    def cmd_paths(self, patch, _prev=tuple()):
         commands = odict()
         for item in patch.itms:
             key, childs, context = item.row, item.child, item.context
+
             if childs:
-                for k, v in self.cmd_paths(childs, _prev + " " + key).items():
+                for k, v in self.cmd_paths(childs, (*_prev, key.strip())).items():
                     commands[k] = v
             else:
-                if key.startswith("delete"):
-                    cmd = "delete" + _prev + " " + key.replace("delete", "", 1).strip()
+                if "comment" in context:
+                    value = (
+                        ""
+                        if key.startswith("delete")
+                        else context["comment"]
+                    )
+
+                    cmd = "\n".join(
+                        (
+                            "edit " + " ".join(_prev),
+                            " ".join(("annotate", context["row"].split(" ")[0], f'"{value}"')),
+                            "exit"
+                        )
+                    )
+                elif key.startswith("delete"):
+                    cmd = " ".join(("delete", *_prev, key.replace("delete", "", 1).strip()))
                 elif key.startswith("activate"):
-                    cmd = "activate" + _prev + " " + key.replace("activate", "", 1).strip()
+                    cmd = " ".join(("activate", *_prev, key.replace("activate", "", 1).strip()))
                 elif key.startswith("deactivate"):
-                    cmd = "deactivate" + _prev + " " + key.replace("deactivate", "", 1).strip()
+                    cmd = " ".join(("deactivate", *_prev, key.replace("deactivate", "", 1).strip()))
                 else:
-                    cmd = (self.patch_set_prefix + _prev.strip()).strip() + " " + key
+                    cmd = " ".join((self.patch_set_prefix, *_prev, key.strip()))
+
                 # Expanding [ a b c ] junipers list of arguments
-                matches = re.search(r"^(.*)\s+\[(.+)\]$", cmd)
-                if matches:
+                if matches := re.search(r"^(.*)\s+\[(.+)\]$", cmd):
                     for c in matches.group(2).split(" "):
                         if c.strip():
                             cmd = " ".join([matches.group(1), c])
@@ -490,7 +544,7 @@ class JuniperList:
 
 
 class NokiaFormatter(JuniperFormatter):
-    patch_set_prefix = "/configure "
+    patch_set_prefix = "/configure"
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -517,18 +571,18 @@ class NokiaFormatter(JuniperFormatter):
         finish = finish if finish is not None else len(ret)
         return ret[start:finish]
 
-    def cmd_paths(self, patch, _prev=""):
+    def cmd_paths(self, patch, _prev=tuple()):
         commands = odict()
         for item in patch.itms:
             key, childs, context = item.row, item.child, item.context
             if childs:
-                for k, v in self.cmd_paths(childs, _prev + " " + key).items():
+                for k, v in self.cmd_paths(childs, (*_prev, key.strip())).items():
                     commands[k] = v
             else:
                 if key.startswith("delete"):
-                    cmd = "/configure delete" + _prev + " " + key.replace("delete", "", 1).strip()
+                    cmd = " ".join((self.patch_set_prefix, "delete", *_prev, key.replace("delete", "", 1).strip()))
                 else:
-                    cmd = self.patch_set_prefix + _prev.strip() + " " + key
+                    cmd = " ".join((self.patch_set_prefix, *_prev, key.strip()))
                 # Expanding [ a b c ] junipers list of arguments
                 matches = re.search(r"^(.*)\s+\[(.+)\]$", cmd)
                 if matches:

annet/executor.py

@@ -1,29 +1,13 @@
 import asyncio
-import logging
-import multiprocessing
 import os
-import platform
-import resource
-import signal
 import statistics
-import time
 from abc import ABC, abstractmethod
 from functools import partial
 from operator import itemgetter
-from queue import Empty
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
 
 import colorama
-import psutil
-
-import annet.lib
 from annet.annlib.command import Command, CommandList, Question  # noqa: F401
-from annet.storage import Device
-
-
-_logger = logging.getLogger(__name__)
-FIRST_EXCEPTION = 1
-ALL_COMPLETED = 2
 
 
 class CommandResult(ABC):
@@ -54,19 +38,6 @@ class Connector(ABC):
         pass
 
 
-class Executor(ABC):
-    # method for bulk config downloading TODO: remove in favor Connector.cmd
-    @abstractmethod
-    def fetch(self,
-              devices: List[Device],
-              files_to_download: Dict[str, List[str]] = None) -> Tuple[Dict[Device, str], Dict[Device, Any]]:
-        pass
-
-    @abstractmethod
-    async def amake_connection(self, device: Device) -> Connector:
-        pass
-
-
 class ExecutorException(Exception):
     def __init__(self, *args: List[Any], auxiliary: Optional[Any] = None, **kwargs: object):
         self.auxiliary = auxiliary
@@ -103,117 +74,6 @@ class NonzeroRetcode(ExecException):
 class CommitException(ExecException):
     pass
 
-def chunks_tuple(l, n):  # noqa
-    return [tuple(l[i:i + n]) for i in range(0, len(l), n)]
-
-
-def async_bulk(
-        executor: Executor,
-        devices: List[Device],
-        coro_gen: Callable[[Connector, Device], Any],
-        *args,
-        processes: int = 1,
-        show_report: bool = True,
-        do_log: bool = True,
-        log_dir: Optional[str] = None,
-        policy: int = ALL_COMPLETED,
-        **kwargs,
-):
-    """Connect to specified devices and work with their CLI.
-
-    Note: this function is not allowed to be run in parallel, since it's using global state (TODO: fixme)
-
-    :param devices: List of devices' fqdns to use their CLI.
-    :param coro_gen: Async function. It contains all logic about usage of CLI.
-        See docstring of "bind_coro_args" for allowed function signature and examples.
-    :param args: Positional arguments to "bulk" function.
-    :type processes: Amount of processes to fork for current work.
-    :param show_report: Set this flag to show report to stdout.
-    :param do_log: If True and log_dir is not set, then log_dir will be filled automatically.
-    :param log_dir: Specify path to log all response from devices.
-    :param policy: int flag. If FIRST_EXCEPTION, then work will be stopped after first error.
-        Otherwise all hosts will be processed.
-        TODO: fix that policy is not used if processes=1
-    :param kwargs: other arguments to pass to "bulk" function.
-        Note: it is not passed directly to "coro_gen" function!
-            kwargs should be {'kwargs': {'var1': value1}} to set "var1" with "value1" in "coro_gen" function.
-
-    TODOs:
-        * do not log if do_log=False and log_dir is set.
-
-    """
-    res = {}
-    deploy_durations = {}
-    kwargs["log_dir"] = log_dir
-    kwargs["policy"] = policy
-
-    if processes == 1:
-        host_res, host_duration = annet.lib.do_async(bulk(executor, devices, coro_gen, *args, **kwargs))
-        res.update(host_res)
-        deploy_durations.update(host_duration)
-    else:
-        # FIXME: show_report works per process
-        if len(devices) != len(set(devices)):
-            raise Exception("hostnames should be unique")
-        # warm up a cache
-        # asyncio.get_event_loop().run_until_complete(get_validator_rt_data(hostnames))
-        if isinstance(devices, dict):
-            devices = list(devices.keys())
-        hostnames_chunks = chunks_tuple(devices, int(len(devices) / processes) + 1)
-        pool = {}
-        for hostnames_chunk in hostnames_chunks:
-            res_q = multiprocessing.Queue()
-            p = multiprocessing.Process(target=_mp_async_bulk, args=[res_q, hostnames_chunk, coro_gen, *args], kwargs=kwargs)
-            pool[p] = [res_q, hostnames_chunk]
-            p.start()
-            _logger.info("process (id=%d) work with %d chunks", p.pid, len(hostnames_chunks))
-
-        seen_error = False
-        while True:
-            done = []
-            for p in pool:
-                host_res = None
-                try:
-                    # proc wont be exited till q.get() call
-                    host_res, host_duration = pool[p][0].get(timeout=0.2)
-                except Empty:
-                    pass
-                else:
-                    done.append(p)
-
-                if not p.is_alive() and not host_res:
-                    _logger.error("process %s has died: hostnames: %s", p.pid, pool[p][1])
-                    host_res = {hostname: Exception("died with exitcode %s" % p.exitcode) for hostname in pool[p][1]}
-                    host_duration = {hostname: 0 for hostname in pool[p][1]}  # FIXME:
-                    done.append(p)
-
-                if host_res:
-                    res.update(host_res)
-                    deploy_durations.update(host_duration)
-
-                if p.exitcode:
-                    _logger.error("process %s finished with bad exitcode %s", p.pid, p.exitcode)
-                    seen_error = True
-            for p in done:
-                pool.pop(p)
-            if policy == FIRST_EXCEPTION and seen_error:
-                for p in pool:
-                    p.terminate()
-                    if p.is_alive():
-                        time.sleep(0.4)
-                    if p.is_alive():
-                        os.kill(p.pid, signal.SIGKILL)
-                    for hostname in pool[p][1]:
-                        res[hostname] = Exception("force kill with exitcode %s" % p.exitcode)
-                        deploy_durations[hostname] = 0  # FIXME:
-            if not pool:
-                break
-
-    if show_report:
-        show_bulk_report(devices, res, deploy_durations, do_log and log_dir)
-
-    return res
-
 
 def _show_type_summary(caption, items, total, stat_items=None):
     if items:
@@ -314,139 +174,6 @@ def _print_failed(host, res):
     print("  %s - %s" % (color + host + colorama.Style.RESET_ALL, _format_exc(exc)))
 
 
-def _mp_async_bulk(res_q: multiprocessing.Queue, *args, **kwargs):
-    res = annet.lib.do_async(bulk(*args, **kwargs))
-    res_q.put(res)
-    res_q.close()
-
-
-async def bulk(
-        executor: Executor,
-        devices: List[Device],
-        coro_gen: Callable[[Connector, Device, Optional[Dict[str, Any]]], Any],
-        max_parallel: float = 100,
-        policy: int = ALL_COMPLETED,
-        log_dir: str = True,  # pylint: disable=unused-argument
-        kwargs: Optional[dict] = None,
-        console_log: bool = True
-) -> Tuple[Dict[str, Any], Dict[str, float]]:
-    """Connect to specified devices and work with their CLI.
-
-    :param hostnames: List of devices' fqdns to use their CLI.
-    :param coro_gen: Async function. It contains all logic about usage of CLI.
-        See docstring of "bind_coro_args" for allowed function signature and examples.
-    :param max_parallel: Upper border to CPU usage (in percentage 1 CPU = 100).
-        If cpu usage is over, then tasks are trottled.
-    :param policy: Flag to specify when tasks are completed.
-    :param log_dir: Specify path to log all response from devices.
-        TODO: fix default value.
-    :param kwargs: Device independent arguments to call function. See @bind_coro_args for details.
-    :param get_device: See "make_connection" for better understanding.
-    :param device_cls: See "make_connection" for better understanding.
-    :param streamer_cls: See "make_connection" for better understanding.
-    :param console_log: If True and there is no handlers for root logger, then stderr will be used for logging.
-    :return: two dicts with results per host and execution duration per host.
-
-    """
-    if console_log:
-        init_log()
-
-    tasks = []
-    res = {}
-    pending = set()
-    tasks_to_device = {}
-    time_of_start = {}
-    deploy_durations = {}
-    now = None
-    if not kwargs:
-        kwargs = {}
-
-    def start_hook(device: Device):
-        time_of_start[device.hostname] = time.monotonic()
-
-    def end_hook(device: Device, task: asyncio.Task):
-        duration = now - time_of_start[device.hostname]
-        deploy_durations[device.hostname] = duration
-
-        coro_exc = task.exception()
-        if coro_exc:
-            if policy == FIRST_EXCEPTION:
-                _logger.error("%s %r", device.hostname, coro_exc, exc_info=coro_exc)
-                _logger.info("Terminating all running tasks according to FIRST_EXCEPTION policy")
-                res[device.hostname] = coro_exc
-                raise CancelAllTasks
-            else:
-                if isinstance(coro_exc, AssertionError):
-                    _logger.error("%s %r", device.hostname, coro_exc, exc_info=coro_exc)
-                else:
-                    _logger.error("%s %r", device.hostname, coro_exc)
-            return coro_exc
-        else:
-            _logger.info("Finished in %0.2fs, hostname=%s", duration, device.hostname)
-            return task.result()
-
-    for device in devices:
-        try:
-            conn = await executor.amake_connection(device=device)
-        except Exception as exc:
-            _logger.error("failed to connect to %s %r", device.hostname, exc)
-            res[device] = exc
-            continue
-        start_hook(device)
-        task = asyncio.create_task(coro_gen(conn=conn, device=device, **kwargs))
-        tasks_to_device[task] = device
-        tasks.append(task)
-    try:
-        ndone = 0
-        with CpuThrottler(asyncio.get_event_loop(), maximum=max_parallel) as throttler:
-            while pending or tasks:
-                left_fds = int(fd_left() / 2)  # better to be safe than sorry
-
-                for _ in range(min(throttler.curr - len(pending), len(tasks), left_fds)):
-                    pending.add(tasks.pop(0))
-                if len(pending) == 0:
-                    _logger.debug("empty pending list. tasks=%s throttler curr=%s left_fds=%s. waiting", len(tasks),
-                                  throttler.curr, left_fds)
-                    await asyncio.sleep(1)
-                    continue
-                example_host = next(iter(pending))
-                _logger.debug("tasks status: %d pending, %d queued, pending example %s", len(pending), len(tasks), example_host)
-                done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
-
-                now = time.monotonic()
-                for task in done:
-                    device = tasks_to_device[task]
-                    res[device] = end_hook(device, task)
-                    ndone += 1
-    except CancelAllTasks:
-        exc = asyncio.CancelledError()
-
-        now = time.monotonic()
-        for device, task in _get_remaining(tasks, pending, tasks_to_device):
-            res[device] = exc
-
-            if device.hostname in time_of_start:
-                duration = now - time_of_start[device.hostname]
-            else:
-                duration = None
-            deploy_durations[device.hostname] = duration
-
-            if not asyncio.iscoroutine(task):
-                _logger.info("task %s", task)
-                task.cancel()
-
-    return res, deploy_durations
-
-
-def init_log():
-    streamer = logging.StreamHandler()
-    fmt = logging.Formatter("%(asctime)s - %(filename)s:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s",
-                            "%Y-%m-%d %H:%M:%S")
-    streamer.setFormatter(fmt)
-    if not logging.root.handlers:
-        logging.root.addHandler(streamer)
-
-
 class DeferredFileWrite:
     def __init__(self, file, mode="r"):
         self._file = file
@@ -465,87 +192,3 @@ class DeferredFileWrite:
 
     def flush(self):
         pass
-
-
-class CancelAllTasks(Exception):
-    pass
-
-
-def _get_remaining(tasks, pending, tasks_to_device):
-    for task in pending:
-        yield (tasks_to_device[task], task)
-    for task in tasks:
-        yield (tasks_to_device[task], task)
-
-
-_platform = platform.system()
-_fd_available = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
-
-
-def fd_left():
-    res = _fd_available
-    if _platform == "Linux":
-        res = _fd_available - len(list(os.scandir(path="/proc/self/fd/")))
-    return res
-
-
-class CpuThrottler:
-    def __init__(self, loop, start=20, maximum=None, minimum=5, hz=1.0, target=80.0):
-        self.loop = loop
-        self.minimum = int(minimum)
-        self.maximum = int(maximum or 0)
-        self.hz = hz
-        self.target = target
-        self.timer_handle = None
-        self.last_usage = 0
-        self.curr = int(start)
-        self.proc = psutil.Process(os.getpid())
-
-    def __enter__(self):
-        if self.proc and self.maximum:
-            self.proc.cpu_percent()  # initialize previous value
-            self.timer_handle = self.loop.call_later(self.hz, self.schedule)
-        return self
-
-    def __exit__(self, type_, value, tb):
-        if self.timer_handle:
-            self.timer_handle.cancel()
-
-    def schedule(self):
-        # re-schedule
-        self.timer_handle = self.loop.call_later(self.hz, self.schedule)
-
-        cpu_usage = self.proc.cpu_percent()
-        self.last_usage = cpu_usage
-        _logger.debug("current cpu_usage=%s", cpu_usage)
-        if cpu_usage > self.target:
-            self.change_by(0.5)
-        elif cpu_usage > self.target * 0.8:
-            pass
-        elif cpu_usage > self.target * 0.2:
-            self.change_by(1.2)
-        else:
-            self.change_by(1.5)
-
-    def change_by(self, rate):
-        new_curr = int(self.curr * rate)
-        # округлим в нужную сторону
-        if new_curr == self.curr:
-            if rate > 1:
-                new_curr += 1
-            elif rate < 1:
-                new_curr -= 1
-        # ограничим пределами
-        if new_curr < self.curr:
-            new_curr = max(self.minimum, new_curr)
-        else:
-            if self.maximum is not None:
-                new_curr = min(self.maximum, new_curr)
-
-        if new_curr < self.curr:
-            _logger.info("decreasing max_slots %d -> %d, cpu_usage=%.1f", self.curr, new_curr, self.last_usage)
-        elif new_curr > self.curr:
-            _logger.info("increasing max_slots %d -> %d, cpu_usage=%.1f", self.curr, new_curr, self.last_usage)
-
-        # new_curr не делаем меньше 0, иначе не сможем его увеличить
-        self.curr = max(new_curr, 1)

annet/rulebook/juniper/__init__.py

@@ -1,73 +1,122 @@
 import re
 from collections import OrderedDict as odict
-from functools import wraps
+import functools
 
 from annet.annlib.lib import jun_activate, jun_is_inactive, merge_dicts
 from annet.annlib.types import Op
-
+from annet.tabparser import JuniperFormatter
 from annet.rulebook import common
 
 
-def _inactive_blocks(diff_foo):
-    @wraps(diff_foo)
-    def wrapper(old, new, diff_pre, *args, **kwargs):
+def comment_processor(item: common.DiffItem):
+    if item.op in (Op.REMOVED, Op.ADDED) and item.row.startswith(JuniperFormatter.Comment.begin):
+        comment = JuniperFormatter.Comment.loads(item.row)
+
+        item.diff_pre["attrs"]["context"]["row"] = comment.row
+        item.diff_pre["attrs"]["context"]["comment"] = comment.comment
+        item.diff_pre["key"] = item.diff_pre["raw_rule"] = (
+            f"{JuniperFormatter.Comment.begin} {comment.row} {JuniperFormatter.Comment.end}"
+        )
+
+        return common.DiffItem(
+            item.op,
+            f"{JuniperFormatter.Comment.begin} {comment.comment} {JuniperFormatter.Comment.end}",
+            item.children,
+            item.diff_pre,
+        )
+    else:
+        return item
+
+
+def inactive_blocks(diff_foo):
+    @functools.wraps(diff_foo)
+    def wrapper(old: odict, new: odict, diff_pre, *args, **kwargs):
         old_inactives = list(map(jun_activate, filter(jun_is_inactive, old)))
         new_inactives = list(map(jun_activate, filter(jun_is_inactive, new)))
+
+        diff: list[common.DiffItem]
+
         if len(old_inactives) == 0 and len(new_inactives) == 0:
-            return diff_foo(old, new, diff_pre, *args, **kwargs)
-
-        inactive_pre = odict([(jun_activate(k), v) for k, v in diff_pre.items() if jun_is_inactive(k)])
-        merged_pre = merge_dicts(diff_pre, inactive_pre)
-        diff = diff_foo(_strip_toplevel_inactives(old),
-                        _strip_toplevel_inactives(new),
-                        merged_pre,
-                        *args, **kwargs)
-
-        for activated in [k for k in old_inactives if k in new]:
-            diff += [(Op.ADDED, _activate_cmd(activated, merged_pre), {}, diff_pre[activated]["match"])]
-
-        for deactivated in [k for k in new_inactives if k not in old_inactives]:
-            # если деактивуруемого блока не существует - ставим один deactivate, глубже не идем
-            if deactivated not in diff_pre:
-                diff = [(Op.ADDED, _deactivate_cmd(deactivated, merged_pre), {}, inactive_pre[deactivated]["match"])]
-            else:
-                diff += [(Op.ADDED, _deactivate_cmd(deactivated, merged_pre), {}, diff_pre[deactivated]["match"])]
-        return diff
+            diff = diff_foo(old, new, diff_pre, *args, **kwargs)
+        else:
+            inactive_pre = odict([(jun_activate(k), v) for k, v in diff_pre.items() if jun_is_inactive(k)])
+            merged_pre = merge_dicts(diff_pre, inactive_pre)
+
+            diff = diff_foo(
+                strip_toplevel_inactives(old),
+                strip_toplevel_inactives(new),
+                merged_pre,
+                *args, **kwargs
+            )
+
+            for activated in [k for k in old_inactives if k in new]:
+                diff.append(
+                    common.DiffItem(
+                        Op.ADDED,
+                        activate_cmd(activated, merged_pre),
+                        [],
+                        diff_pre[activated]["match"]
+                    )
+                )
+
+            for deactivated in [k for k in new_inactives if k not in old_inactives]:
+                # если деактивуруемого блока не существует - ставим один deactivate, глубже не идем
+                if deactivated not in diff_pre:
+                    diff = [
+                        common.DiffItem(
+                            Op.ADDED,
+                            deactivate_cmd(deactivated, merged_pre),
+                            [],
+                            inactive_pre[deactivated]["match"]
+                        )
+                    ]
+                else:
+                    diff.append(
+                        common.DiffItem(
+                            Op.ADDED,
+                            deactivate_cmd(deactivated, merged_pre),
+                            [],
+                            diff_pre[deactivated]["match"]
+                        )
+                    )
+
+        return list(map(comment_processor, diff))
+
     return wrapper
 
 
-@_inactive_blocks
+@inactive_blocks
 def default_diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
     diff = common.default_diff(old, new, diff_pre, _pops)
-    diff = _ignore_quotes(diff)
-    diff = _strip_inactive_removed(diff)
+    diff = ignore_quotes(diff)
+    diff = strip_inactive_removed(diff)
     return diff
 
 
-@_inactive_blocks
+@inactive_blocks
 def ordered_diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
     diff = common.ordered_diff(old, new, diff_pre, _pops)
-    diff = _ignore_quotes(diff)
-    diff = _strip_inactive_removed(diff)
+    diff = ignore_quotes(diff)
+    diff = strip_inactive_removed(diff)
     return diff
 
 
 # =====
-def _strip_toplevel_inactives(tree):
+def strip_toplevel_inactives(tree):
     for inactive in filter(jun_is_inactive, tree):
         assert jun_activate(inactive) not in tree
     return odict([(k, v) if not jun_is_inactive(k) else (jun_activate(k), v) for k, v in tree.items()])
 
 
-def _activate_cmd(active_key, diff_pre):
-    return _cmd(active_key, diff_pre, "activate")
+def activate_cmd(active_key, diff_pre):
+    return cmd(active_key, diff_pre, "activate")
 
 
-def _deactivate_cmd(active_key, diff_pre):
-    return _cmd(active_key, diff_pre, "deactivate")
+def deactivate_cmd(active_key, diff_pre):
+    return cmd(active_key, diff_pre, "deactivate")
 
 
-def _cmd(active_key, diff_pre, cmd):
+def cmd(active_key, diff_pre, cmd):
     assert not jun_is_inactive(active_key)
     if not diff_pre[active_key]["subtree"]:
         # Если конанда не имеет подблоков И имеет агрументы то надо их отбросить
@@ -75,7 +124,7 @@ def _cmd(active_key, diff_pre, cmd):
     return " ".join([cmd, active_key])
 
 
-def _ignore_quotes(diff):
+def ignore_quotes(diff):
     """
     Фильтрует из diff строки которые различаются
     только наличием/отсутсвием кавычек
@@ -86,19 +135,19 @@ def _ignore_quotes(diff):
     """
     equivs = {}
     for elem in diff:
-        key = _strip_quotes(elem[1])
+        key = strip_quotes(elem[1])
         if key not in equivs:
             equivs[key] = 0
         equivs[key] += 1
-    filtered_diff = [elem for elem in diff if equivs[_strip_quotes(elem[1])] == 1]
+    filtered_diff = [elem for elem in diff if equivs[strip_quotes(elem[1])] == 1]
     return filtered_diff
 
 
-def _strip_quotes(key):
+def strip_quotes(key):
     return re.sub(r"\"(?P<quoted_text>[^\"]+)\"$", r"\g<quoted_text>", key)
 
 
-def _strip_inactive_removed(diff):
+def strip_inactive_removed(diff):
     for elem in diff:
         if elem[0] == Op.REMOVED and elem[3]["key"]:
             key = elem[3]["key"][0]

annet/rulebook/texts/arista.order

@@ -1,3 +1,4 @@
+# vim: set syntax=annrulebook:
 # В этом файле определяется порядок команд, в котором их следует подавать на устройство.
 # - Если порядок команды не важен - ее можно не писать сюда совсем.
 # - Если команда начинается с undo и прописан параметр %order_reverse - команда считается
@@ -29,7 +30,7 @@ ntp
 
 
 qos profile *
-    ~ %global
+    ~
 
 
 snmp-server
@@ -49,7 +50,7 @@ tacacs-server
 
 
 aaa group
-    ~ %global
+    ~
 
 
 aaa
@@ -59,7 +60,7 @@ username
 
 
 role
-    ~ %global
+    ~
 
 
 vlan
@@ -73,14 +74,14 @@ mpls ip
 mpls ldp
 
 router isis
-    ~ %global
+    ~
 
 interface
-    ~ %global
+    ~
 
 
 (?:ip|ipv6) access-list
-    ~ %global
+    ~
 
 
 (?:ip|ipv6) route
@@ -101,15 +102,15 @@ qos map
 
 
 route-map * (?:permit|deny) *
-    ~ %global
+    ~
 
 
 router bfd
-    ~ %global
+    ~
 
 
 router bgp
-    ~ %global
+    ~
 
 
 router multicast
@@ -119,8 +120,7 @@ ip tacacs
 
 
 management
-    ~ %global
+    ~
 
 
-# vim: set syntax=annrulebook:
-~ %global
+~

annet/rulebook/texts/huawei.order

@@ -1,3 +1,4 @@
+# vim: set syntax=annrulebook:
 # В этом файле определяется порядок команд, в котором их следует подавать на устройство.
 # - Если порядок команды не важен - ее можно не писать сюда совсем.
 # - Если команда начинается с undo и прописан параметр %order_reverse - команда считается
@@ -384,5 +385,4 @@ undo sflow %order_reverse
 # индекс референсится в логике патчинга префикс-листа huawei.misc.prefix_list
 undo ip */(ip|ipv6)/-prefix * index 99999999  %order_reverse
 
-# vim: set syntax=annrulebook:
-~ %global
+~

annet/rulebook/texts/juniper.order

@@ -0,0 +1,4 @@
+# Правило на все команды
+* %global
+# Добавляем веса комментам, чтобы они катились всегда после команд
+*/\/\*(?:(?!\*\/).)*\*\// %global

{annet-1.0.4.dist-info → annet-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: annet
-Version: 1.0.4
+Version: 1.1.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -15,7 +15,6 @@ Requires-Dist: PyYAML>=6.0.1
 Requires-Dist: Pygments>=2.14.0
 Requires-Dist: Mako>=1.2.4
 Requires-Dist: Jinja2>=3.1.2
-Requires-Dist: psutil>=5.8.0
 Requires-Dist: packaging>=23.2
 Requires-Dist: contextlog>=1.1
 Requires-Dist: valkit>=0.1.4

{annet-1.0.4.dist-info → annet-1.1.0.dist-info}/RECORD

@@ -8,7 +8,7 @@ annet/connectors.py,sha256=aoiDVLPizx8CW2p8SAwGCzyO_WW8H9xc2aujbGC4bDg,4882
 annet/deploy.py,sha256=3O96k17FbVt8KCvxF4gujXAB81U2-XRJyHLpbc9ekSQ,7529
 annet/deploy_ui.py,sha256=SDTJ-CF6puW0KHQ0g_NDp61Tqh6xkTBMxv8PrBhGyNI,27977
 annet/diff.py,sha256=zLcaCnb4lZRUb7frpH1CstQ3kacRcCblZs1uLG8J5lk,3391
-annet/executor.py,sha256=Jny-hm0otZA1naPpFWR-R16SbaZioSQ8pkx-Yd2PYlM,19004
+annet/executor.py,sha256=ZOuexyr2k7CtUttaP1mAZMRoSA-ja_AHaH6inQji24A,5575
 annet/filtering.py,sha256=ZtqxPsKdV9reZoRxtQyBg22BqyMqd-2SotYcxZ-68AQ,903
 annet/gen.py,sha256=A718tYqIcxAa8tQEdjR6PjQ2ovWBnwPH7STKh38lmFY,33567
 annet/hardware.py,sha256=_iR28dWiPtt6ZYdk-qg1sxazkSRJE3ukqKB-fFFfQak,1141
@@ -50,8 +50,8 @@ annet/annlib/filter_acl.py,sha256=0w1VF6WcONiTYTQh0yWi6_j9rCTc_kMLAUMr0hbdkNU,72
 annet/annlib/jsontools.py,sha256=BS7s4rI8R9c_y3zz0zYl1l6con65oQ0MvfsC1dsXZts,5535
 annet/annlib/lib.py,sha256=eJ4hcVuQ6pdYBzLs4YKCHFtq45idOfZCYp92XfF7_QI,15317
 annet/annlib/output.py,sha256=_SjJ6G6bejvnTKqNHw6xeio0FT9oO3OIkLaOC3cEga4,7569
-annet/annlib/patching.py,sha256=p5u3jl3_Iod0CGcnfZsfFR3Izo_roorny_v0stjDCWs,21142
-annet/annlib/tabparser.py,sha256=Xsje7t2bEZqZ8hhgnEYgjQGaDZ5mBWgNxwE2wpCkwXQ,28961
+annet/annlib/patching.py,sha256=5CZOT5RZsOgOPm-NhzJe3YwpbYlw6DLKsOhvZil6pdc,21258
+annet/annlib/tabparser.py,sha256=oGQyf_w6y4mnms0Lybjw1LDf3vD1DuucV00SHYWrpKI,30833
 annet/annlib/types.py,sha256=VHU0CBADfYmO0xzB_c5f-mcuU3dUumuJczQnqGlib9M,852
 annet/annlib/netdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 annet/annlib/netdev/db.py,sha256=fI_u5aya4l61mbYSjj4JwlVfi3s7obt2jqERSuXGRUI,1634
@@ -63,8 +63,8 @@ annet/annlib/netdev/views/hardware.py,sha256=3JCZLH7deIHhCguwPJTUX-WDvWjG_xt6BdS
 annet/annlib/rbparser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 annet/annlib/rbparser/acl.py,sha256=RR8yPt6t96_IiyuKVbeZ-3x32cyhBAT2wC1y99oWBO8,3931
 annet/annlib/rbparser/deploying.py,sha256=ACT8QNhDAhJx3ZKuGh2nYBOrpdka2qEKuLDxvQAGKLk,1649
-annet/annlib/rbparser/ordering.py,sha256=DiKqyY8Khz-5MTxNF1GSNtZgtyKwT3YYCXpahIPB6Ps,1779
-annet/annlib/rbparser/platform.py,sha256=hnxznTfV9txXi1PkR1hZrprTrQJvlwgqXVL8vXkYmv4,1558
+annet/annlib/rbparser/ordering.py,sha256=Ux_uIOywzSbYcALK_k_bdzhC9pyYerqETsI63hsutJI,1949
+annet/annlib/rbparser/platform.py,sha256=Q9HtqmhyzV3JK_236_4LjC2wgp5fgxY6seDfWYl1oHU,1558
 annet/annlib/rbparser/syntax.py,sha256=iZ7Y-4QQBw4L3UtjEh54qisiRDhobl7HZxFNdP8mi54,3577
 annet/annlib/rulebook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 annet/annlib/rulebook/common.py,sha256=hqwmmNofm5q2f-hV2usMY-IPMeiANLth28tZcRBYJTw,16640
@@ -131,14 +131,13 @@ annet/rulebook/huawei/bgp.py,sha256=dN8T3-44ggGEapt4u4sT3bTn_dsoCbS5qdNeSQ8LSTs,
 annet/rulebook/huawei/iface.py,sha256=DvLtQ7tfbDQWFmIYV4lxfih13Tdrt24L4-ZS29mCkuc,1126
 annet/rulebook/huawei/misc.py,sha256=Rpwhtm42IgcueDq4K6VOzN2qORoIDYh42Jb7iWL8AII,14424
 annet/rulebook/huawei/vlandb.py,sha256=B4BEUhZetjsNNhIJOp9cXtJSAYKMOgQucO8oAxSkRI0,4658
-annet/rulebook/juniper/__init__.py,sha256=WByFrqKCdr-ORCme2AURQGnYbN66t3omB3-9dWktT9o,3887
+annet/rulebook/juniper/__init__.py,sha256=FnVaO7FHYz7O1ExRE6igYX7OKMUgx5_siC-aoHOLx1s,5380
 annet/rulebook/nexus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 annet/rulebook/nexus/iface.py,sha256=aeog9iSGT2zZ78tsGlrRcfgfOv7yW3jLwryXqdeplRw,2923
-annet/rulebook/ribbon/__init__.py,sha256=TRbkQVvk0-HxkUQW9-LmiG6XIfTZ-8t3SiaDemNCzK4,347
 annet/rulebook/routeros/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 annet/rulebook/routeros/file.py,sha256=zK7RwBk1YaVoDSFSg1u7Pt8u0Fk3nhhu27aJRngemwc,137
 annet/rulebook/texts/arista.deploy,sha256=OS9eFyJpEPztcOHkBwajw_RTJfTT7ivaMHfx4_HXaUg,792
-annet/rulebook/texts/arista.order,sha256=e7e5pnzMZAfmXJ0sUaaWmep6JMeErNnd7DkouUVIZsU,1472
+annet/rulebook/texts/arista.order,sha256=1jbKfF0USYi2b8V5qFlp4LjBAr3QGL9OkmoQqEK3PW8,1385
 annet/rulebook/texts/arista.rul,sha256=fGZe1L-x75rKQuRC5oLJMv1gSt1tJ0htOXfKt-tho64,853
 annet/rulebook/texts/aruba.deploy,sha256=hI432Bq-of_LMXuUflCu7eNSEFpx6qmj0KItEw6sgHI,841
 annet/rulebook/texts/aruba.order,sha256=ZMakkn0EJ9zomgY6VssoptJImrHrUmYnCqivzLBFTRo,1158
@@ -150,8 +149,9 @@ annet/rulebook/texts/cisco.deploy,sha256=Hu0NkcGv3f1CWUrnbzI3eQOPXJxtH4NNOPRV68I
 annet/rulebook/texts/cisco.order,sha256=OvNHMNqkCc-DN2dEjLCTKv_7ZhiaHt4q2X4Y4Z8dvR4,1901
 annet/rulebook/texts/cisco.rul,sha256=jgL5_xnSwd_H4E8cx4gcneSvJC5W1zz6_BWSb64iuxI,3017
 annet/rulebook/texts/huawei.deploy,sha256=uUsZCHUrC5Zyb_MePrR5svnE1QyGQlg7UxcKf00sJyg,10451
-annet/rulebook/texts/huawei.order,sha256=ENllPX4kO6xNw2mUQcx11yhxo3tKStZ5mUyc0C6s3d0,10657
+annet/rulebook/texts/huawei.order,sha256=_X2LmxD9FGcIcb8ziwc13Imy1tBtQJgafrQGmvHPZLM,10649
 annet/rulebook/texts/huawei.rul,sha256=02Fi1RG4YYea2clHCluBuJDKNbT0hS9jtsk6_h6GK8k,12958
+annet/rulebook/texts/juniper.order,sha256=yUid8mNGQtZSq_WUZisuBtKtdWSKu4_MPVRIHS1neN0,210
 annet/rulebook/texts/juniper.rul,sha256=EmtrEJZesnmc2nXjURRD2G0WOq4zLluI_PNupKhSOJs,2654
 annet/rulebook/texts/nexus.deploy,sha256=9YNAQEw7aQxtYZJbE-dMD6qJrTzs_G92Ifrx3Ft4Wn4,1120
 annet/rulebook/texts/nexus.order,sha256=AZMKCD5Zf_mBOlE36aMDvO4w5rdbepTz1Dsyv7xP9Qs,1834
@@ -178,10 +178,10 @@ annet_generators/rpl_example/generator.py,sha256=zndIGfV4ZlTxPgAGYs7bMQvTc_tYScO
 annet_generators/rpl_example/items.py,sha256=Ez1RF5YhcXNCusBmeApIjRL3rBlMazNZd29Gpw1_IsA,766
 annet_generators/rpl_example/mesh.py,sha256=z_WgfDZZ4xnyh3cSf75igyH09hGvtexEVwy1gCD_DzA,288
 annet_generators/rpl_example/route_policy.py,sha256=z6nPb0VDeQtKD1NIg9sFvmUxBD5tVs2frfNIuKdM-5c,2318
-annet-1.0.4.dist-info/AUTHORS,sha256=rh3w5P6gEgqmuC-bw-HB68vBCr-yIBFhVL0PG4hguLs,878
-annet-1.0.4.dist-info/LICENSE,sha256=yPxl7dno02Pw7gAcFPIFONzx_gapwDoPXsIsh6Y7lC0,1079
-annet-1.0.4.dist-info/METADATA,sha256=X4P627F_6XM4xksFYwLxf9vqw-u7ou3br5iUkqwdFaA,851
-annet-1.0.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-annet-1.0.4.dist-info/entry_points.txt,sha256=5lIaDGlGi3l6QQ2ry2jZaqViP5Lvt8AmsegdD0Uznck,192
-annet-1.0.4.dist-info/top_level.txt,sha256=QsoTZBsUtwp_FEcmRwuN8QITBmLOZFqjssRfKilGbP8,23
-annet-1.0.4.dist-info/RECORD,,
+annet-1.1.0.dist-info/AUTHORS,sha256=rh3w5P6gEgqmuC-bw-HB68vBCr-yIBFhVL0PG4hguLs,878
+annet-1.1.0.dist-info/LICENSE,sha256=yPxl7dno02Pw7gAcFPIFONzx_gapwDoPXsIsh6Y7lC0,1079
+annet-1.1.0.dist-info/METADATA,sha256=sIW7Dw8SLb9IBLPciWoWXTUlq0lx6n_lRsFhTyzOtk0,822
+annet-1.1.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+annet-1.1.0.dist-info/entry_points.txt,sha256=5lIaDGlGi3l6QQ2ry2jZaqViP5Lvt8AmsegdD0Uznck,192
+annet-1.1.0.dist-info/top_level.txt,sha256=QsoTZBsUtwp_FEcmRwuN8QITBmLOZFqjssRfKilGbP8,23
+annet-1.1.0.dist-info/RECORD,,

{annet-1.0.4.dist-info → annet-1.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

annet/rulebook/ribbon/__init__.py

@@ -1,12 +0,0 @@
-from annet.annlib.rulebook import common
-from annet.annlib.types import Op
-
-
-def default_diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
-    diff = common.default_diff(old, new, diff_pre, _pops)
-    return diff
-
-
-def ordered_diff(old, new, diff_pre, _pops=(Op.AFFECTED,)):
-    diff = common.ordered_diff(old, new, diff_pre, _pops)
-    return diff