annet 1.1.2__py3-none-any.whl → 2.0.1__py3-none-any.whl

annet/adapters/netbox/provider.py

@@ -28,11 +28,12 @@ def storage_factory(opts: NetboxStorageOpts) -> Storage:
 
 class NetboxProvider(StorageProvider, AdapterWithName, AdapterWithConfig):
     def __init__(self, url: Optional[str] = None, token: Optional[str] = None, insecure: bool = False,
-                 exact_host_filter: bool = False):
+                 exact_host_filter: bool = False, threads: int = 1):
         self.url = url
         self.token = token
         self.insecure = insecure
         self.exact_host_filter = exact_host_filter
+        self.threads = threads
 
     @classmethod
     def with_config(cls, **kwargs: Dict[str, Any]) -> T:

annet/annet.py

@@ -2,7 +2,7 @@
 import sys
 
 import annet
-from annet import argparse, cli, generators, hardware, lib, rulebook
+from annet import argparse, cli, generators, hardware, lib, rulebook, diff
 
 
 # =====
@@ -13,6 +13,7 @@ def main():
     cli.fill_base_args(parser, annet.__name__, "configs/logging.yaml")
     rulebook.rulebook_provider_connector.set(rulebook.DefaultRulebookProvider)
     hardware.hardware_connector.set(hardware.AnnetHardwareProvider)
+    diff.file_differ_connector.set(diff.UnifiedFileDiffer)
 
     parser.add_commands(parser.find_subcommands(cli.list_subcommands()))
     try:

annet/annlib/filter_acl.py

@@ -32,7 +32,6 @@ def filter_config(acl: Acl, fmtr: tabparser.CommonFormatter, input_config: Input
         config = patching.apply_acl(config, acl, fatal_acl=False)
         config = fmtr.join(config)
     else:
-        config = typing.cast(input_config, FileConfigTree)
         config = apply_acl_fileconfig(input_config, acl)
     return config
 
@@ -47,7 +46,6 @@ def filter_diff(acl: Acl, fmtr: tabparser.CommonFormatter, input_config: InputCo
         config = unshift_op(config)
         config = config.rstrip()
     else:
-        config = typing.cast(input_config, FileConfigTree)
         config = apply_acl_fileconfig(input_config, acl)
     return config
 
@@ -108,17 +106,23 @@ def get_op(line: str) -> typing.Tuple[str, str, str]:
     indent = ""
     opidx = -1
     rowstart = 0
+
     for rowstart in range(len(line)):
+        if line[rowstart] != " " and line[0:rowstart].strip():
+            break
         if line[rowstart] not in diff_ops:
             break
+
     for opidx in range(rowstart):
         if line[opidx] != " ":
             break
+
     if opidx >= 0:
         op = line[opidx]
         indent = line[:opidx] + line[opidx + 1:rowstart]
     if op != " ":
         indent = indent + " "
+
     return op, indent, line[rowstart:]
 
 

annet/api/__init__.py

@@ -1,5 +1,4 @@
 import abc
-import difflib
 import os
 import re
 import sys
@@ -37,6 +36,7 @@ from annet import diff as ann_diff
 from annet import filtering
 from annet import gen as ann_gen
 from annet import patching, rulebook, tabparser, tracing
+from annet.diff import file_differ_connector
 from annet.rulebook import deploying
 from annet.filtering import Filterer
 from annet.hardware import hardware_connector
@@ -52,8 +52,6 @@ from annet.storage import Device, get_storage
 from annet.types import Diff, ExitCode, OldNewResult, Op, PCDiff, PCDiffFile
 
 
-live_configs = ann_gen.live_configs
-
 DEFAULT_INDENT = "  "
 
 
@@ -242,29 +240,22 @@ def gen(args: cli_args.ShowGenOptions, loader: ann_gen.Loader):
 
 
 # =====
-def _diff_file(old_text: Optional[str], new_text: Optional[str], context=3):
-    old_lines = old_text.splitlines() if old_text else []
-    new_lines = new_text.splitlines() if new_text else []
-    context = max(len(old_lines), len(new_lines)) if context is None else context
-    return list(difflib.unified_diff(old_lines, new_lines, n=context, lineterm=""))
-
-
-def _diff_files(old_files, new_files, context=3):
+def _diff_files(hw, old_files, new_files):
     ret = {}
+    differ = file_differ_connector.get()
     for (path, (new_text, reload_data)) in new_files.items():
         old_text = old_files.get(path)
         is_new = old_text is None
-        diff_lines = _diff_file(old_text, new_text, context=context)
+        diff_lines = differ.diff_file(hw, path, old_text, new_text)
         ret[path] = (diff_lines, reload_data, is_new)
     return ret
 
 
 def patch(args: cli_args.ShowPatchOptions, loader: ann_gen.Loader):
     """ Сгенерировать патч для устройств """
-    global live_configs  # pylint: disable=global-statement
     if args.config == "running":
         fetcher = annet.deploy.get_fetcher()
-        live_configs = annet.lib.do_async(fetcher.fetch(loader.devices, processes=args.parallel))
+        ann_gen.live_configs = annet.lib.do_async(fetcher.fetch(loader.devices, processes=args.parallel))
     stdin = args.stdin(filter_acl=args.filter_acl, config=args.config)
 
     filterer = filtering.filterer_connector.get()
@@ -355,9 +346,9 @@ def diff(
 
         pc_diff_files = []
         if res.old_files or new_files:
-            pc_diff_files.extend(_pc_diff(device.hostname, res.old_files, new_files))
+            pc_diff_files.extend(_pc_diff(res.device.hw, device.hostname, res.old_files, new_files))
         if res.old_json_fragment_files or new_json_fragment_files:
-            pc_diff_files.extend(_json_fragment_diff(device.hostname, res.old_json_fragment_files, new_json_fragment_files))
+            pc_diff_files.extend(_json_fragment_diff(res.device.hw, device.hostname, res.old_json_fragment_files, new_json_fragment_files))
 
         if pc_diff_files:
             pc_diff_files.sort(key=lambda f: f.label)
@@ -478,18 +469,19 @@ class PCDeployerJob(DeployerJob):
         upload_files: Dict[str, bytes] = {}
         reload_cmds: Dict[str, bytes] = {}
         generator_types: Dict[str, GeneratorType] = {}
+        differ = file_differ_connector.get()
         for generator_type, pc_files in [(GeneratorType.ENTIRE, new_files), (GeneratorType.JSON_FRAGMENT, new_json_fragment_files)]:
             for file, (file_content_or_json_cfg, cmds) in pc_files.items():
                 if generator_type == GeneratorType.ENTIRE:
                     file_content: str = file_content_or_json_cfg
-                    diff_content = "\n".join(_diff_file(old_files.get(file), file_content))
+                    diff_content = "\n".join(differ.diff_file(res.device.hw, file, old_files.get(file), file_content))
                 else:  # generator_type == GeneratorType.JSON_FRAGMENT
                     old_json_cfg = old_json_fragment_files[file]
                     json_patch = jsontools.make_patch(old_json_cfg, file_content_or_json_cfg)
                     file_content = jsontools.format_json(json_patch)
                     old_text = jsontools.format_json(old_json_cfg)
                     new_text = jsontools.format_json(file_content_or_json_cfg)
-                    diff_content = "\n".join(_diff_file(old_text, new_text))
+                    diff_content = "\n".join(differ.diff_file(res.device.hw, file, old_text, new_text))
 
                 if diff_content or force_reload:
                     self._has_diff |= True
@@ -624,7 +616,6 @@ class Deployer:
         return ans
 
     def check_diff(self, result: annet.deploy.DeployResult, loader: ann_gen.Loader):
-        global live_configs  # pylint: disable=global-statement
         success_device_ids = []
         for host, hres in result.results.items():
             device = self.fqdn_to_device[host]
@@ -639,7 +630,7 @@ class Deployer:
             config="running",
         )
         if diff_args.query:
-            live_configs = None
+            ann_gen.live_configs = None
             diffs = diff(diff_args, loader, success_device_ids, self._filterer)
             non_pc_diffs = {dev: diff for dev, diff in diffs.items() if not isinstance(diff, PCDiff)}
             devices_to_diff = ann_diff.collapse_diffs(non_pc_diffs)
@@ -682,13 +673,23 @@ async def adeploy(
 ) -> ExitCode:
     """ Сгенерировать конфиг для устройств и задеплоить его """
     ret: ExitCode = 0
-    global live_configs  # pylint: disable=global-statement
-    live_configs = await fetcher.fetch(devices=loader.devices, processes=args.parallel)
-    pool = ann_gen.OldNewParallel(args, loader, filterer)
+    ann_gen.live_configs = await fetcher.fetch(devices=loader.devices, processes=args.parallel)
 
-    for res in pool.generated_configs(loader.devices):
+    device_ids = [d.id for d in loader.devices]
+    for res in ann_gen.old_new(
+        args,
+        config=args.config,
+        loader=loader,
+        no_new=args.clear,
+        stdin=args.stdin(filter_acl=args.filter_acl, config=args.config),
+        do_files_download=True,
+        device_ids=device_ids,
+        filterer=filterer,
+    ):
         # Меняем exit code если хоть один device ловил exception
         if res.err is not None:
+            if not args.tolerate_fails:
+                raise res.err
             get_logger(res.device.hostname).error("error generating configs", exc_info=res.err)
             ret |= 2 ** 3
         job = DeployerJob.from_device(res.device, args)
@@ -746,12 +747,16 @@ def file_diff(args: cli_args.FileDiffOptions):
 def file_diff_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions) -> Generator[
     Tuple[str, str, bool], None, None]:
     old_path, new_path = old_new
+    hw = args.hw
+    if isinstance(args.hw, str):
+        hw = HardwareView(args.hw, "")
+
     if os.path.isdir(old_path) and os.path.isdir(new_path):
         hostname = os.path.basename(new_path)
         new_files = {relative_cfg_path: (cfg_text, "") for relative_cfg_path, cfg_text in
                      ann_gen.load_pc_config(new_path).items()}
         old_files = ann_gen.load_pc_config(old_path)
-        for diff_file in _pc_diff(hostname, old_files, new_files):
+        for diff_file in _pc_diff(hw, hostname, old_files, new_files):
             diff_text = (
                 "\n".join(diff_file.diff_lines)
                 if args.no_color
@@ -791,8 +796,8 @@ def file_patch_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions)
             yield dest_name, patch_text, False
 
 
-def _pc_diff(hostname: str, old_files: Dict[str, str], new_files: Dict[str, str]) -> Generator[PCDiffFile, None, None]:
-    sorted_lines = sorted(_diff_files(old_files, new_files).items())
+def _pc_diff(hw, hostname: str, old_files: Dict[str, str], new_files: Dict[str, str]) -> Generator[PCDiffFile, None, None]:
+    sorted_lines = sorted(_diff_files(hw, old_files, new_files).items())
     for (path, (diff_lines, _reload_data, is_new)) in sorted_lines:
         if not diff_lines:
             continue
@@ -803,6 +808,7 @@ def _pc_diff(hostname: str, old_files: Dict[str, str], new_files: Dict[str, str]
 
 
 def _json_fragment_diff(
+        hw,
         hostname: str,
         old_files: Dict[str, Any],
         new_files: Dict[str, Tuple[Any, Optional[str]]],
@@ -820,7 +826,7 @@ def _json_fragment_diff(
             ret[path] = (jsontools.format_json(cfg), reload_cmd)
         return ret
     jold, jnew = jsonify_multi(old_files), jsonify_multi_with_cmd(new_files)
-    return _pc_diff(hostname, jold, jnew)
+    return _pc_diff(hw, hostname, jold, jnew)
 
 
 def guess_hw(config_text: str):

annet/diff.py

@@ -1,6 +1,9 @@
+import abc
+import difflib
 import re
 from itertools import groupby
-from typing import Generator, List, Mapping, Tuple, Union
+from pathlib import Path
+from typing import Generator, List, Mapping, Tuple, Union, Protocol
 
 from annet.annlib.diff import (  # pylint: disable=unused-import
     colorize_line,
@@ -9,10 +12,12 @@ from annet.annlib.diff import (  # pylint: disable=unused-import
     gen_pre_as_diff,
     resort_diff,
 )
+from annet.annlib.netdev.views.hardware import HardwareView
 from annet.annlib.output import format_file_diff
 
-from annet import patching
+from annet import patching, rulebook, tabparser, hardware
 from annet.cli_args import ShowDiffOptions
+from annet.connectors import CachedConnector
 from annet.output import output_driver_connector
 from annet.storage import Device
 from annet.tabparser import make_formatter
@@ -82,3 +87,67 @@ def collapse_diffs(diffs: Mapping[Device, Diff]) -> Mapping[Tuple[Device, ...],
         res[tuple(x[0] for x in collapsed_diff)] = collapsed_diff[0][1][0]
 
     return res
+
+
+class FileDiffer(Protocol):
+    @abc.abstractmethod
+    def diff_file(self, hw: HardwareView, path: str | Path, old: str, new: str) -> list[str]:
+        raise NotImplementedError
+
+
+class UnifiedFileDiffer(FileDiffer):
+    def __init__(self):
+        self.context: int = 3
+
+    def diff_file(self, hw: HardwareView, path: str | Path, old: str, new: str) -> list[str]:
+        """Calculate the differences for config files.
+
+        Args:
+            hw: device hardware info
+            path: path to file on a device
+            old (Optional[str]): The old file content.
+            new (Optional[str]): The new file content.
+
+        Returns:
+            List[str]: List of difference lines.
+        """
+        return self._diff_text_file(old, new)
+
+    def _diff_text_file(self, old, new):
+        """Calculate the differences for plaintext files."""
+        context = self.context
+        old_lines = old.splitlines() if old else []
+        new_lines = new.splitlines() if new else []
+        context = max(len(old_lines), len(new_lines)) if context is None else context
+        return list(difflib.unified_diff(old_lines, new_lines, n=context, lineterm=""))
+
+
+class FrrFileDiffer(UnifiedFileDiffer):
+    def diff_file(self, hw: HardwareView, path: str | Path, old: str, new: str) -> list[str]:
+        if (hw.PC.Mellanox or hw.PC.NVIDIA) and (path == "/etc/frr/frr.conf"):
+            return self._diff_frr_conf(hw, old, new)
+        return super().diff_file(hw, path, old, new)
+
+    def _diff_frr_conf(self,  hw: HardwareView, old_text: str | None, new_text: str | None) -> list[str]:
+        """Calculate the differences for frr.conf files."""
+        indent = "  "
+        rb = rulebook.rulebook_provider_connector.get()
+        rulebook_data = rb.get_rulebook(hw)
+        formatter = tabparser.make_formatter(hw, indent=indent)
+
+        old_tree = tabparser.parse_to_tree(old_text or "", splitter=formatter.split)
+        new_tree = tabparser.parse_to_tree(new_text or "", splitter=formatter.split)
+
+        diff_tree = patching.make_diff(old_tree, new_tree, rulebook_data, [])
+        pre_diff = patching.make_pre(diff_tree)
+        diff_iterator = gen_pre_as_diff(pre_diff, show_rules=False, indent=indent, no_color=True)
+
+        return [line.rstrip() for line in diff_iterator if "frr version" not in line]
+
+
+class _FileDifferConnector(CachedConnector[FileDiffer]):
+    name = "Device file diff processor"
+    ep_name = "file_differ"
+
+
+file_differ_connector = _FileDifferConnector()

annet/gen.py

@@ -526,52 +526,6 @@ def worker(device_id, args: ShowGenOptions, stdin, loader: "Loader", filterer: F
                    False)
 
 
-def old_new_worker(device_id, args: DeployOptions, config, stdin, loader: "Loader", filterer: Filterer):
-    for res in old_new(
-        args,
-        config=config,
-        loader=loader,
-        filterer=filterer,
-        stdin=stdin,
-        device_ids=[device_id],
-        no_new=args.clear,
-        do_files_download=True,
-    ):
-        if res.err is not None and not args.tolerate_fails:
-            raise res.err
-        yield res
-
-
-class OldNewParallel(Parallel):
-    def __init__(self, args: DeployOptions, loader: "Loader", filterer: Filterer):
-        stdin = args.stdin(filter_acl=args.filter_acl, config=args.config)
-        super().__init__(
-            old_new_worker,
-            args,
-            config=args.config,
-            stdin=stdin,
-            loader=loader,
-            filterer=filterer,
-        )
-        self.tune_args(args)
-        self.tolerate_fails = args.tolerate_fails
-
-    def generated_configs(self, devices: List[Device]) -> Generator[OldNewResult, None, None]:
-        devices_by_id = {device.id: device for device in devices}
-        device_ids = list(devices_by_id)
-
-        for task_result in self.irun(device_ids, self.tolerate_fails):
-            if task_result.exc is not None:
-                device = devices_by_id.pop(task_result.device_id)
-                yield OldNewResult(device=device, err=task_result.exc)
-            elif task_result.result is not None:
-                yield from task_result.result
-                devices_by_id.pop(task_result.device_id)
-
-        for device in devices_by_id.values():
-            yield OldNewResult(device=device, err=Exception(f"No config returned for {device.hostname}"))
-
-
 @dataclasses.dataclass
 class DeviceFilesToDownload:
     entire: List[str] = dataclasses.field(default_factory=list)

annet/output.py

@@ -155,8 +155,15 @@ class OutputDriverBasic(OutputDriver):
             ret.append((label, getattr(exc, "formatted_output", f"{repr(exc)} (formatted_output is absent)"), True))
         return ret
 
-    def cfg_file_names(self, device: Device) -> List[str]:
-        return [f"{device.hostname}.cfg"]
+    def cfg_file_names(self, device: Device) -> list[str]:
+        res = []
+        if device.hostname:
+            res.append(f"{device.hostname}.cfg")
+        if device.id is not None and device.id != "":
+            res.append(f"_id_{device.id}.cfg")
+        if not res:
+            raise RuntimeError("Neither hostname nor id is known for device")
+        return res
 
     def entire_config_dest_path(self, device, config_path: str) -> str:
         """Формирует путь к конфигу в директории destname.

annet/rpl/__init__.py

@@ -16,11 +16,12 @@ __all__ = [
     "RoutingPolicy",
     "CommunityActionValue",
     "PrefixMatchValue",
+    "OrLonger",
 ]
 
 from .action import Action, ActionType, SingleAction
 from .condition import AndCondition, Condition, ConditionOperator, SingleCondition
-from .match_builder import R, MatchField, PrefixMatchValue
+from .match_builder import R, MatchField, PrefixMatchValue, OrLonger
 from .policy import RoutingPolicyStatement, RoutingPolicy
 from .result import ResultType
 from .routemap import RouteMap, Route

annet/rpl/match_builder.py

@@ -63,11 +63,15 @@ class SetConditionFactory(Generic[ValueT]):
         return SingleCondition(self.field, ConditionOperator.HAS_ANY, values)
 
 
+# OrLonger represents a pair of (le, ge)
+# for prefix mask length match in prefix-lists
+OrLonger = tuple[Optional[int], Optional[int]]
+
+
 @dataclass(frozen=True)
 class PrefixMatchValue:
     names: tuple[str, ...]
-    greater_equal: Optional[int]
-    less_equal: Optional[int]
+    or_longer: OrLonger = (None, None)
 
 
 class Checkable:
@@ -91,23 +95,23 @@ class Checkable:
     def match_v6(
             self,
             *names: str,
-            or_longer: tuple[Optional[int], Optional[int]] = (None, None),
+            or_longer: OrLonger = (None, None),
     ) -> SingleCondition[PrefixMatchValue]:
         return SingleCondition(
             MatchField.ipv6_prefix,
             ConditionOperator.CUSTOM,
-            PrefixMatchValue(names, greater_equal=or_longer[0], less_equal=or_longer[1]),
+            PrefixMatchValue(names, or_longer),
         )
 
     def match_v4(
             self,
             *names: str,
-            or_longer: tuple[Optional[int], Optional[int]] = (None, None),
+            or_longer: OrLonger = (None, None),
     ) -> SingleCondition[PrefixMatchValue]:
         return SingleCondition(
             MatchField.ip_prefix,
             ConditionOperator.CUSTOM,
-            PrefixMatchValue(names, greater_equal=or_longer[0], less_equal=or_longer[1]),
+            PrefixMatchValue(names, or_longer),
         )
 
 

annet/rpl/statement_builder.py

@@ -1,3 +1,4 @@
+import warnings
 from collections.abc import Callable
 from dataclasses import dataclass
 from dataclasses import field
@@ -14,6 +15,7 @@ class ThenField(str, Enum):
     large_community = "large_community"
     extcommunity_rt = "extcommunity_rt"
     extcommunity_soo = "extcommunity_soo"
+    extcommunity = "extcommunity"
     as_path = "as_path"
     local_pref = "local_pref"
     metric = "metric"
@@ -144,6 +146,7 @@ class StatementBuilder:
         self._added_as_path: list[int] = []
         self._community = CommunityActionValue()
         self._large_community = CommunityActionValue()
+        self._extcommunity = CommunityActionValue()
         self._extcommunity_rt = CommunityActionValue()
         self._extcommunity_soo = CommunityActionValue()
         self._as_path = AsPathActionValue()
@@ -165,12 +168,18 @@ class StatementBuilder:
     def large_community(self) -> CommunityActionBuilder:
         return CommunityActionBuilder(self._large_community)
 
+    @property
+    def extcommunity(self) -> CommunityActionBuilder:
+        return CommunityActionBuilder(self._extcommunity)
+
     @property
     def extcommunity_rt(self) -> CommunityActionBuilder:
+        warnings.warn("extcommunity_rt is deprecated, use extcommunity", DeprecationWarning, stacklevel=2)
         return CommunityActionBuilder(self._extcommunity_rt)
 
     @property
     def extcommunity_soo(self) -> CommunityActionBuilder:
+        warnings.warn("extcommunity_soo is deprecated, use extcommunity", DeprecationWarning, stacklevel=2)
         return CommunityActionBuilder(self._extcommunity_soo)
 
     def _set(self, field: str, value: ValueT) -> None:
@@ -244,7 +253,13 @@ class StatementBuilder:
             self._statement.then.append(SingleAction(
                 field=ThenField.large_community,
                 type=ActionType.CUSTOM,
-                value=self._extcommunity_rt,
+                value=self._large_community,
+            ))
+        if self._extcommunity:
+            self._statement.then.append(SingleAction(
+                field=ThenField.extcommunity,
+                type=ActionType.CUSTOM,
+                value=self._extcommunity,
             ))
         if self._extcommunity_rt:
             self._statement.then.append(SingleAction(
@@ -256,7 +271,7 @@ class StatementBuilder:
             self._statement.then.append(SingleAction(
                 field=ThenField.extcommunity_soo,
                 type=ActionType.CUSTOM,
-                value=self._extcommunity_rt,
+                value=self._extcommunity_soo,
             ))
         if self._as_path:
             self._statement.then.append(SingleAction(

annet/rpl_generators/__init__.py

@@ -10,14 +10,16 @@ __all__ = [
     "RDFilterFilterGenerator",
     "RDFilter",
     "IpPrefixList",
+    "IpPrefixListMember",
     "PrefixListFilterGenerator",
     "get_policies",
+    "ip_prefix_list",
 ]
 
 from .aspath import AsPathFilterGenerator
 from .community import CommunityListGenerator
 from .cumulus_frr import CumulusPolicyGenerator
-from .entities import CommunityList, AsPathFilter, CommunityType, CommunityLogic, RDFilter, IpPrefixList
+from .entities import CommunityList, AsPathFilter, CommunityType, CommunityLogic, RDFilter, IpPrefixList, IpPrefixListMember, ip_prefix_list
 from .execute import get_policies
 from .policy import RoutingPolicyGenerator
 from .prefix_lists import PrefixListFilterGenerator