annet 2.2.1__tar.gz → 2.3.0__tar.gz

{annet-2.2.1/annet.egg-info → annet-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: annet
-Version: 2.2.1
+Version: 2.3.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

{annet-2.2.1 → annet-2.3.0}/annet/api/__init__.py

@@ -1,4 +1,5 @@
 import abc
+import inspect
 import os
 import re
 import sys
@@ -8,13 +9,11 @@ from collections import OrderedDict as odict
 from itertools import groupby, chain
 from operator import itemgetter
 from typing import (
-    Any,
     Dict,
     Generator,
     Iterable,
     List,
     Mapping,
-    Optional,
     Set,
     Tuple,
     Union, cast,
@@ -41,7 +40,6 @@ from annet.rulebook import deploying
 from annet.filtering import Filterer
 from annet.hardware import hardware_connector
 from annet.output import (
-    LABEL_NEW_PREFIX,
     format_file_diff,
     output_driver_connector,
     print_err_label,
@@ -51,7 +49,6 @@ from annet.reference import RefTracker
 from annet.storage import Device, get_storage
 from annet.types import Diff, ExitCode, OldNewResult, Op, PCDiff, PCDiffFile
 
-
 DEFAULT_INDENT = "  "
 
 
@@ -240,15 +237,6 @@ def gen(args: cli_args.ShowGenOptions, loader: ann_gen.Loader):
 
 
 # =====
-def _diff_files(hw, old_files, new_files):
-    ret = {}
-    differ = file_differ_connector.get()
-    for (path, (new_text, reload_data)) in new_files.items():
-        old_text = old_files.get(path)
-        is_new = old_text is None
-        diff_lines = differ.diff_file(hw, path, old_text, new_text)
-        ret[path] = (diff_lines, reload_data, is_new)
-    return ret
 
 
 def patch(args: cli_args.ShowPatchOptions, loader: ann_gen.Loader):
@@ -322,62 +310,39 @@ def res_diff_patch(
 
 
 def diff(
-        args: cli_args.DiffOptions,
-        loader: ann_gen.Loader,
-        device_ids: List[int],
-        filterer: filtering.Filterer,
+    args: cli_args.DiffOptions,
+    loader: ann_gen.Loader,
 ) -> Mapping[Device, Union[Diff, PCDiff]]:
-    ret = {}
-    for res in ann_gen.old_new(
-        args,
-        config=args.config,
-        loader=loader,
-        no_new=args.clear,
-        do_files_download=True,
-        device_ids=device_ids,
-        filterer=filterer,
-    ):
-        old = res.get_old(args.acl_safe)
-        new = res.get_new(args.acl_safe)
-        device = res.device
-        acl_rules = res.get_acl_rules(args.acl_safe)
-        new_files = res.get_new_files(args.acl_safe)
-        new_json_fragment_files = res.get_new_file_fragments(args.acl_safe)
-
-        pc_diff_files = []
-        if res.old_files or new_files:
-            pc_diff_files.extend(_pc_diff(res.device.hw, device.hostname, res.old_files, new_files))
-        if res.old_json_fragment_files or new_json_fragment_files:
-            pc_diff_files.extend(_json_fragment_diff(res.device.hw, device.hostname, res.old_json_fragment_files, new_json_fragment_files))
+    """ Сгенерировать конфиг для устройств """
+    stdin = args.stdin(filter_acl=args.filter_acl, config=None)
 
-        if pc_diff_files:
-            pc_diff_files.sort(key=lambda f: f.label)
-            ret[device] = PCDiff(hostname=device.hostname, diff_files=pc_diff_files)
-        elif old is not None:
-            orderer = patching.Orderer.from_hw(device.hw)
-            rb = rulebook.get_rulebook(device.hw)
-            diff_tree = patching.make_diff(
-                old,
-                orderer.order_config(new),
-                rb,
-                [acl_rules, res.filter_acl_rules],
-            )
-            diff_tree = patching.strip_unchanged(diff_tree)
-            ret[device] = diff_tree
+    filterer = filtering.filterer_connector.get()
+    pool = Parallel(ann_diff.worker, args, stdin, loader, filterer).tune_args(args)
+    if args.show_hosts_progress:
+        pool.add_callback(PoolProgressLogger(loader.device_fqdns))
 
-    return ret
+    return pool.run(loader.device_ids, args.tolerate_fails, args.strict_exit_code)
 
 
-def collapse_texts(texts: Mapping[str, str]) -> Mapping[Tuple[str, ...], str]:
+def collapse_texts(texts: Mapping[str, str | Generator[str, None, None]]) -> Mapping[Tuple[str, ...], str]:
     """
     Группировка текстов.
     :param texts:
     :return: словарь с несколькими хостнеймами в ключе.
     """
-    diffs_with_orig = {key: [value, value.splitlines()] for key, value in texts.items()}
+    diffs_with_orig = {}
+    for key, value in texts.items():
+        if inspect.isgenerator(value):
+            lines = list(value)
+            diffs_with_orig[key] = ["".join(lines), lines]
+        else:
+            diffs_with_orig[key] = [value, value.splitlines()]
+
     res = {}
-    for _, collapsed_diff_iter in groupby(sorted(diffs_with_orig.items(), key=lambda x: (x[0], x[1][1])),
-                                          key=lambda x: x[1][1]):
+    for _, collapsed_diff_iter in groupby(
+        sorted(diffs_with_orig.items(), key=lambda x: (x[0], x[1][1])),
+        key=lambda x: x[1][1]
+    ):
         collapsed_diff = list(collapsed_diff_iter)
         res[tuple(x[0] for x in collapsed_diff)] = collapsed_diff[0][1][0]
 
@@ -744,8 +709,10 @@ def file_diff(args: cli_args.FileDiffOptions):
     return pool.run(old_new, tolerate_fails=True)
 
 
-def file_diff_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions) -> Generator[
-    Tuple[str, str, bool], None, None]:
+def file_diff_worker(
+    old_new: Tuple[str, str],
+    args: cli_args.FileDiffOptions
+) -> Generator[Tuple[str, str, bool], None, None]:
     old_path, new_path = old_new
     hw = args.hw
     if isinstance(args.hw, str):
@@ -753,10 +720,12 @@ def file_diff_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions) -
 
     if os.path.isdir(old_path) and os.path.isdir(new_path):
         hostname = os.path.basename(new_path)
-        new_files = {relative_cfg_path: (cfg_text, "") for relative_cfg_path, cfg_text in
-                     ann_gen.load_pc_config(new_path).items()}
+        new_files = {
+            relative_cfg_path: (cfg_text, "")
+            for relative_cfg_path, cfg_text in ann_gen.load_pc_config(new_path).items()
+        }
         old_files = ann_gen.load_pc_config(old_path)
-        for diff_file in _pc_diff(hw, hostname, old_files, new_files):
+        for diff_file in ann_diff.pc_diff(hw, hostname, old_files, new_files):
             diff_text = (
                 "\n".join(diff_file.diff_lines)
                 if args.no_color
@@ -796,39 +765,6 @@ def file_patch_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions)
             yield dest_name, patch_text, False
 
 
-def _pc_diff(hw, hostname: str, old_files: Dict[str, str], new_files: Dict[str, str]) -> Generator[PCDiffFile, None, None]:
-    sorted_lines = sorted(_diff_files(hw, old_files, new_files).items())
-    for (path, (diff_lines, _reload_data, is_new)) in sorted_lines:
-        if not diff_lines:
-            continue
-        label = hostname + os.sep + path
-        if is_new:
-            label = LABEL_NEW_PREFIX + label
-        yield PCDiffFile(label=label, diff_lines=diff_lines)
-
-
-def _json_fragment_diff(
-        hw,
-        hostname: str,
-        old_files: Dict[str, Any],
-        new_files: Dict[str, Tuple[Any, Optional[str]]],
-) -> Generator[PCDiffFile, None, None]:
-    def jsonify_multi(files):
-        return {
-            path: jsontools.format_json(cfg)
-            for path, cfg in files.items()
-        }
-
-    def jsonify_multi_with_cmd(files):
-        ret = {}
-        for path, cfg_reload_cmd in files.items():
-            cfg, reload_cmd = cfg_reload_cmd
-            ret[path] = (jsontools.format_json(cfg), reload_cmd)
-        return ret
-    jold, jnew = jsonify_multi(old_files), jsonify_multi_with_cmd(new_files)
-    return _pc_diff(hw, hostname, jold, jnew)
-
-
 def guess_hw(config_text: str):
     """Пытаемся угадать вендора и hw на основе
     текста конфига и annet/rulebook/texts/*.rul"""

{annet-2.2.1 → annet-2.3.0}/annet/cli.py

@@ -113,6 +113,7 @@ def show_device_dump(args: cli_args.QueryOptions, arg_out: cli_args.FileOutOptio
                 )
             else:
                 get_logger().warning("method `dump` not implemented for %s", type(device))
+
     arg_gens = cli_args.GenOptions(arg_out, args)
     with get_loader(arg_gens, args) as loader:
         if not loader.devices:
@@ -190,7 +191,7 @@ def gen(args: cli_args.ShowGenOptions):
         output_driver = output_driver_connector.get()
         if args.dest is None:
             text_mapping = {item[0]: item[1] for item in out}
-            out = [(",".join(key), value, False) for key, value in collapse_texts(text_mapping).items()]
+            out = [(", ".join(key), value, False) for key, value in collapse_texts(text_mapping).items()]
 
         out.extend(output_driver.format_fails(fail, loader.device_fqdns))
         total = len(success) + len(fail)
@@ -203,13 +204,19 @@ def gen(args: cli_args.ShowGenOptions):
 def diff(args: cli_args.ShowDiffOptions):
     """ Generate configuration for devices and show a diff with current configuration using the rulebook """
     with get_loader(args, args) as loader:
-        filterer = filtering.filterer_connector.get()
-        device_ids = loader.device_ids
-        output_driver_connector.get().write_output(
-            args,
-            gen_sort_diff(api.diff(args, loader, device_ids, filterer), args),
-            len(loader.device_ids)
-        )
+        success, fail = api.diff(args, loader)
+
+        out = list(gen_sort_diff({loader.get_device(k): v for k, v in success.items()}, args))
+        output_driver = output_driver_connector.get()
+        if args.dest is None:
+            text_mapping = {item[0]: item[1] for item in out}
+            out = [(", ".join(key), value, False) for key, value in collapse_texts(text_mapping).items()]
+        out.extend(output_driver.format_fails(fail, loader.device_fqdns))
+
+        if total := len(success) + len(fail):
+            output_driver.write_output(args, out, total)
+        else:
+            get_logger().error("No devices found for %s", args.query)
 
 
 @subcommand(cli_args.ShowPatchOptions)

{annet-2.2.1 → annet-2.3.0}/annet/cli_args.py

@@ -442,6 +442,7 @@ class FileOutOptions(ArgGroup):
 class DiffOptions(GenOptions, ComocutorOptions):
     clear = opt_clear
     config = opt_config
+    show_hosts_progress = opt_show_hosts_progress
 
 
 class FileInputOptions(ArgGroup):

{annet-2.2.1 → annet-2.3.0}/annet/deploy_ui.py

@@ -782,11 +782,10 @@ def draw_lines_in_win(
     max_x -= 2 * (margin or x_margin)
     lines_count = len(lines)
     if lines_count > max_y:
-        start_line = lines_count - max_y + 1
+        start_line = lines_count - max_y
     else:
         start_line = 0
     for line_no, line_data in enumerate(lines[start_line:]):
-        line_no = line_no - start_line
         line_pos_calc = 0
         for line_part in line_data:
             if not isinstance(line_part, TextArgs):

{annet-2.2.1 → annet-2.3.0}/annet/diff.py

@@ -1,10 +1,13 @@
 import abc
 import difflib
+import os
 import re
 from itertools import groupby
 from pathlib import Path
-from typing import Generator, List, Mapping, Tuple, Union, Protocol
+from typing import Any, Dict, Generator, List, Mapping, Optional, Protocol, Tuple, Union
 
+from annet import cli_args, filtering, patching, rulebook, tabparser
+from annet.annlib import jsontools
 from annet.annlib.diff import (  # pylint: disable=unused-import
     colorize_line,
     diff_cmp,
@@ -13,23 +16,113 @@ from annet.annlib.diff import (  # pylint: disable=unused-import
     resort_diff,
 )
 from annet.annlib.netdev.views.hardware import HardwareView
-from annet.annlib.output import format_file_diff
-
-from annet import patching, rulebook, tabparser, hardware
+from annet.annlib.output import LABEL_NEW_PREFIX, format_file_diff
 from annet.cli_args import ShowDiffOptions
 from annet.connectors import CachedConnector
 from annet.output import output_driver_connector
 from annet.storage import Device
 from annet.tabparser import make_formatter
-from annet.types import Diff, PCDiff
+from annet.types import Diff, PCDiff, PCDiffFile
+
+from .gen import Loader, old_new
+
 
+def _diff_files(hw, old_files, new_files):
+    ret = {}
+    differ = file_differ_connector.get()
+    for (path, (new_text, reload_data)) in new_files.items():
+        old_text = old_files.get(path)
+        is_new = old_text is None
+        diff_lines = differ.diff_file(hw, path, old_text, new_text)
+        ret[path] = (diff_lines, reload_data, is_new)
+    return ret
 
-# NOCDEV-1720
+
+def pc_diff(hw, hostname: str, old_files: Dict[str, str], new_files: Dict[str, str]) -> Generator[
+    PCDiffFile, None, None]:
+    sorted_lines = sorted(_diff_files(hw, old_files, new_files).items())
+    for (path, (diff_lines, _reload_data, is_new)) in sorted_lines:
+        if not diff_lines:
+            continue
+        label = hostname + os.sep + path
+        if is_new:
+            label = LABEL_NEW_PREFIX + label
+        yield PCDiffFile(label=label, diff_lines=diff_lines)
+
+
+def json_fragment_diff(
+    hw,
+    hostname: str,
+    old_files: Dict[str, Any],
+    new_files: Dict[str, Tuple[Any, Optional[str]]],
+) -> Generator[PCDiffFile, None, None]:
+    def jsonify_multi(files):
+        return {
+            path: jsontools.format_json(cfg)
+            for path, cfg in files.items()
+        }
+
+    def jsonify_multi_with_cmd(files):
+        ret = {}
+        for path, cfg_reload_cmd in files.items():
+            cfg, reload_cmd = cfg_reload_cmd
+            ret[path] = (jsontools.format_json(cfg), reload_cmd)
+        return ret
+
+    jold, jnew = jsonify_multi(old_files), jsonify_multi_with_cmd(new_files)
+    return pc_diff(hw, hostname, jold, jnew)
+
+
+def worker(
+    device_id,
+    args: cli_args.DiffOptions,
+    stdin,
+    loader: Loader,
+    filterer: filtering.Filterer,
+) -> Union[Diff, PCDiff, None]:
+    for res in old_new(
+        args,
+        config=args.config,
+        loader=loader,
+        no_new=args.clear,
+        do_files_download=True,
+        device_ids=[device_id],
+        filterer=filterer,
+        stdin=stdin,
+    ):
+        old = res.get_old(args.acl_safe)
+        new = res.get_new(args.acl_safe)
+        device = res.device
+        acl_rules = res.get_acl_rules(args.acl_safe)
+        new_files = res.get_new_files(args.acl_safe)
+        new_json_fragment_files = res.get_new_file_fragments(args.acl_safe)
+
+        pc_diff_files = []
+        if res.old_files or new_files:
+            pc_diff_files.extend(pc_diff(res.device.hw, device.hostname, res.old_files, new_files))
+        if res.old_json_fragment_files or new_json_fragment_files:
+            pc_diff_files.extend(json_fragment_diff(res.device.hw, device.hostname, res.old_json_fragment_files,
+                                                    new_json_fragment_files))
+
+        if pc_diff_files:
+            pc_diff_files.sort(key=lambda f: f.label)
+            return PCDiff(hostname=device.hostname, diff_files=pc_diff_files)
+        elif old is not None:
+            orderer = patching.Orderer.from_hw(device.hw)
+            rb = rulebook.get_rulebook(device.hw)
+            diff_tree = patching.make_diff(
+                old,
+                orderer.order_config(new),
+                rb,
+                [acl_rules, res.filter_acl_rules],
+            )
+            diff_tree = patching.strip_unchanged(diff_tree)
+            return diff_tree
 
 
 def gen_sort_diff(
     diffs: Mapping[Device, Union[Diff, PCDiff]], args: ShowDiffOptions
-) -> Generator[Tuple[str, Generator[str, None, None], bool], None, None]:
+) -> Generator[Tuple[str, Generator[str, None, None] | str, bool], None, None]:
     """
     Возвращает осортированный дифф, совместимый с write_output
     :param diffs: Маппинг устройства в дифф
@@ -71,7 +164,7 @@ def _make_text_diff(device: Device, diff: Diff) -> List[str]:
     return res
 
 
-def collapse_diffs(diffs: Mapping[Device, Diff]) -> Mapping[Tuple[Device, ...], Diff]:
+def collapse_diffs(diffs: Mapping[Device, Diff]) -> Dict[Tuple[Device, ...], Diff]:
     """
     Группировка диффов.
     :param diffs:
@@ -128,7 +221,7 @@ class FrrFileDiffer(UnifiedFileDiffer):
             return self._diff_frr_conf(hw, old, new)
         return super().diff_file(hw, path, old, new)
 
-    def _diff_frr_conf(self,  hw: HardwareView, old_text: str | None, new_text: str | None) -> list[str]:
+    def _diff_frr_conf(self, hw: HardwareView, old_text: str | None, new_text: str | None) -> list[str]:
         """Calculate the differences for frr.conf files."""
         indent = "  "
         rb = rulebook.rulebook_provider_connector.get()

{annet-2.2.1 → annet-2.3.0/annet.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: annet
-Version: 2.2.1
+Version: 2.3.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT