annet 3.2.0__tar.gz → 3.4.0__tar.gz

{annet-3.2.0/annet.egg-info → annet-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.2.0
+Version: 3.4.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -21,7 +21,7 @@ Requires-Dist: yarl>=1.8.2
 Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.3.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.4.0; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: license-file

{annet-3.2.0 → annet-3.4.0}/annet/adapters/netbox/v42/models.py

@@ -3,6 +3,7 @@ from datetime import datetime, timezone
 from typing import Optional
 from annet.adapters.netbox.common.models import InterfaceType, IpFamily, Label, Prefix, Entity
 from annet.adapters.netbox.v41.models import InterfaceV41, IpAddressV41, NetboxDeviceV41
+import annetbox.v42.models
 
 
 @dataclass
@@ -61,3 +62,8 @@ class NetboxDeviceV42(NetboxDeviceV41):
             connected_endpoints=[],
             mode=None,
         )
+
+
+# should be unified in case of netbox update
+Vrf = annetbox.v42.models.Vrf
+Vlan = annetbox.v42.models.Vlan

{annet-3.2.0 → annet-3.4.0}/annet/adapters/netbox/v42/storage.py

@@ -1,4 +1,6 @@
 import ssl
+from typing import Optional
+
 from adaptix import P
 from adaptix.conversion import get_converter, link, link_constant, link_function
 from annetbox.v42 import client_sync
@@ -6,7 +8,8 @@ from annetbox.v42 import models as api_models
 
 from annet.adapters.netbox.common.adapter import NetboxAdapter, get_device_breed, get_device_hw
 from annet.adapters.netbox.common.storage_base import BaseNetboxStorage
-from annet.adapters.netbox.v42.models import InterfaceV42, NetboxDeviceV42, PrefixV42, IpAddressV42
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.adapters.netbox.v42.models import InterfaceV42, NetboxDeviceV42, PrefixV42, IpAddressV42, Vlan, Vrf
 from annet.storage import Storage
 
 
@@ -79,8 +82,21 @@ class NetboxV42Adapter(NetboxAdapter[NetboxDeviceV42, InterfaceV42, IpAddressV42
     def list_ipprefixes(self, prefixes: list[str]) -> list[PrefixV42]:
         return self.convert_ip_prefixes(self.netbox.ipam_all_prefixes(prefix=prefixes).results)
 
+    def list_all_vrfs(self) -> list[Vrf]:
+        return self.netbox.ipam_all_vrfs().results
+
+    def list_all_vlans(self) -> list[Vlan]:
+        return self.netbox.ipam_all_vlans().results
+
 
 class NetboxStorageV42(BaseNetboxStorage[NetboxDeviceV42, InterfaceV42, IpAddressV42, PrefixV42]):
+    netbox: NetboxV42Adapter
+
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        super().__init__(opts)
+        self._all_vlans: list[Vlan] | None = None
+        self._all_vrfs: list[Vrf] | None = None
+
     def _init_adapter(
             self,
             url: str,
@@ -89,3 +105,13 @@ class NetboxStorageV42(BaseNetboxStorage[NetboxDeviceV42, InterfaceV42, IpAddres
             threads: int,
     ) -> NetboxAdapter[NetboxDeviceV42, InterfaceV42, IpAddressV42, PrefixV42]:
         return NetboxV42Adapter(self, url, token, ssl_context, threads)
+
+    def resolve_all_vlans(self) -> list[Vlan]:
+        if self._all_vlans is None:
+            self._all_vlans = self.netbox.list_all_vlans()
+        return self._all_vlans
+
+    def resolve_all_vrfs(self) -> list[Vrf]:
+        if self._all_vrfs is None:
+            self._all_vrfs = self.netbox.list_all_vrfs()
+        return self._all_vrfs

{annet-3.2.0 → annet-3.4.0}/annet/api/__init__.py

@@ -6,9 +6,10 @@ import sys
 import time
 import warnings
 from collections import OrderedDict as odict
-from itertools import groupby, chain
+from itertools import chain, groupby
 from operator import itemgetter
 from typing import (
+    Any,
     Dict,
     Generator,
     Iterable,
@@ -16,39 +17,37 @@ from typing import (
     Mapping,
     Set,
     Tuple,
-    Union, cast, Any,
+    Union,
+    cast,
 )
 
 import colorama
+from contextlog import get_logger
+
 import annet.deploy
 import annet.deploy_ui
 import annet.lib
-from annet.annlib import jsontools
-from annet.annlib.netdev.views.hardware import HardwareView
-from annet.annlib.types import GeneratorType
-from contextlog import get_logger
-
-from annet.deploy import Fetcher, DeployDriver
 from annet import cli_args
 from annet import diff as ann_diff
 from annet import filtering
 from annet import gen as ann_gen
 from annet import patching, rulebook, tabparser, tracing
+from annet.annlib import jsontools
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.annlib.types import GeneratorType
+from annet.deploy import DeployDriver, Fetcher
 from annet.diff import file_differ_connector
-from annet.rulebook import deploying
 from annet.filtering import Filterer
 from annet.hardware import hardware_connector
-from annet.output import (
-    format_file_diff,
-    output_driver_connector,
-    print_err_label,
-)
+from annet.output import format_file_diff, output_driver_connector, print_err_label
 from annet.parallel import Parallel, TaskResult
 from annet.reference import RefTracker
+from annet.rulebook import deploying
 from annet.storage import Device, get_storage
-from annet.types import Diff, ExitCode, OldNewResult, Op, PCDiff, PCDiffFile
+from annet.types import Diff, ExitCode, OldNewResult, Op, PCDiff
 from annet.vendors import registry_connector
 
+
 DEFAULT_INDENT = "  "
 
 
@@ -96,21 +95,45 @@ def _read_old_new_diff_patch(old: Dict[str, Dict], new: Dict[str, Dict], hw: Har
     return rb, diff_obj, pre, patchtree
 
 
-def _read_old_new_hw(old_path: str, new_path: str, args: cli_args.FileInputOptions):
+def _read_old_new_configs(old_path, new_path):
+    _logger = get_logger()
+    with open(old_path) as f:
+        _logger.debug("Reading %r ...", old_path)
+        old_config = f.read()
+
+    with open(new_path) as f:
+        _logger.debug("Reading %r ...", new_path)
+        new_config = f.read()
+
+    return old_config, new_config
+
+
+def _read_old_new_hw(old_path: str, old_config: str, new_path: str, new_config: str, args: cli_args.FileInputOptions):
     _logger = get_logger()
+
     hw = args.hw
     if isinstance(args.hw, str):
         hw = HardwareView(args.hw, "")
 
-    old, old_hw, old_score = _read_device_config(old_path, hw)
-    new, new_hw, new_score = _read_device_config(new_path, hw)
+    try:
+        old, old_hw, old_score = _parse_device_config(old_config, hw)
+    except tabparser.ParserError:
+        _logger.exception("Parser error: %r", old_path)
+        raise
+
+    try:
+        new, new_hw, new_score = _parse_device_config(new_config, hw)
+    except tabparser.ParserError:
+        _logger.exception("Parser error: %r", new_path)
+        raise
+
     hw = new_hw
     if old_score > new_score:
         hw = old_hw
     if old_hw != new_hw:
-        _logger.debug("Old and new detected hw differs, assume %r", hw)
-    dest_name = os.path.basename(new_path)
-    return dest_name, old, new, hw
+        _logger.warning("Old and new detected hw differs, assume %r", hw)
+
+    return old, new, hw
 
 
 @tracing.function
@@ -134,27 +157,19 @@ def _read_old_new_cfgdumps(args: cli_args.FileInputOptions):
         yield (old_path_name, new_path_name)
 
 
-def _read_device_config(path, hw):
-    _logger = get_logger()
-    _logger.debug("Reading %r ...", path)
+def _parse_device_config(text, hw):
     score = 1
     vendor_registry = registry_connector.get()
 
-    with open(path) as cfgdump_file:
-        text = cfgdump_file.read()
-    try:
-        if not hw:
-            hw, score = guess_hw(text)
+    if not hw:
+        hw, score = guess_hw(text)
 
-        config = tabparser.parse_to_tree(
-            text=text,
-            splitter=vendor_registry.match(hw).make_formatter().split,
-        )
+    config = tabparser.parse_to_tree(
+        text=text,
+        splitter=vendor_registry.match(hw).make_formatter().split,
+    )
 
-        return config, hw, score
-    except tabparser.ParserError:
-        _logger.exception("Parser error: %r", path)
-        raise
+    return config, hw, score
 
 
 # =====
@@ -756,12 +771,15 @@ def file_diff_worker(
             if diff_text:
                 yield diff_file.label, diff_text, False
     else:
-        dest_name, old, new, hw = _read_old_new_hw(old_path, new_path, args)
+        old_config, new_config = _read_old_new_configs(old_path, new_path)
+        if old_config == new_config:
+            return
+
+        old, new, hw = _read_old_new_hw(old_path, old_config, new_path, new_config, args)
         _, __, pre, ___ = _read_old_new_diff_patch(old, new, hw, add_comments=False)
-        diff_lines = ann_diff.gen_pre_as_diff(pre, args.show_rules, args.indent, args.no_color)
-        diff_text = "".join(diff_lines)
-        if diff_text:
-            yield dest_name, diff_text, False
+
+        if diff_lines := ann_diff.gen_pre_as_diff(pre, args.show_rules, args.indent, args.no_color):
+            yield os.path.basename(new_path), "".join(diff_lines), False
 
 
 @tracing.function
@@ -773,18 +791,23 @@ def file_patch(args: cli_args.FilePatchOptions):
 
 
 def file_patch_worker(old_new: Tuple[str, str], args: cli_args.FileDiffOptions) -> Generator[
-    Tuple[str, str, bool], None, None]:
+    Tuple[str, str, bool], None, None
+]:
     old_path, new_path = old_new
     if os.path.isdir(old_path) and os.path.isdir(new_path):
         for relative_cfg_path, cfg_text in ann_gen.load_pc_config(new_path).items():
             label = os.path.join(os.path.basename(new_path), relative_cfg_path)
             yield label, cfg_text, False
     else:
-        dest_name, old, new, hw = _read_old_new_hw(old_path, new_path, args)
+        old_config, new_config = _read_old_new_configs(old_path, new_path)
+        if old_config == new_config:
+            return
+
+        old, new, hw = _read_old_new_hw(old_path, old_config, new_path, new_config, args)
         _, __, ___, patch_tree = _read_old_new_diff_patch(old, new, hw, args.add_comments)
         patch_text = _format_patch_blocks(patch_tree, hw, args.indent)
         if patch_text:
-            yield dest_name, patch_text, False
+            yield os.path.basename(new_path), patch_text, False
 
 
 def guess_hw(config_text: str):

{annet-3.2.0 → annet-3.4.0}/annet/generators/__init__.py

@@ -157,7 +157,7 @@ def run_partial_generators(
 
 
 @tracing.function(name="run_partial_generator")
-def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRunArgs) -> Optional[GeneratorPartialResult]:
+def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRunArgs) -> GeneratorPartialResult | None:
     logger = get_logger(generator=_make_generator_ctx(gen))
     device = run_args.device
     output = ""
@@ -207,7 +207,9 @@ def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRu
 
     if run_args.use_acl:
         try:
-            with tracing_connector.get().start_as_current_span("apply_acl", tracer_name=__name__, min_duration="0.01") as acl_span:
+            with tracing_connector.get().start_as_current_span(
+                "apply_acl", tracer_name=__name__, min_duration="0.01"
+            ) as acl_span:
                 tracing_connector.get().set_device_attributes(acl_span, run_args.device)
                 config = patching.apply_acl(
                     config=config,
@@ -265,8 +267,8 @@ def check_entire_generators_required_packages(gens, device_packages: FrozenSet[s
 
 @tracing.function
 def run_file_generators(
-        gens: Iterable[Union["JSONFragment", "Entire"]],
-        device: "Device",
+    gens: Iterable[Union["JSONFragment", "Entire"]],
+    device: "Device",
 ) -> RunGeneratorResult:
     """Run generators that generate files or file parts."""
     ret = RunGeneratorResult()
@@ -295,21 +297,22 @@ def run_file_generators(
 @tracing.function(min_duration="0.5")
 def _run_entire_generator(gen: "Entire", device: "Device") -> Optional[GeneratorResult]:
     logger = get_logger(generator=_make_generator_ctx(gen))
-    if not gen.supports_device(device):
-        logger.info("generator %s is not supported for device %s", gen, device.hostname)
-        return
-
     span = tracing_connector.get().get_current_span()
     if span:
         tracing_connector.get().set_device_attributes(span, device)
         tracing_connector.get().set_dimensions_attributes(span, gen, device)
 
-    path = gen.path(device)
-    if not path:
-        raise RuntimeError("entire generator should return non-empty path")
-
-    logger.info("Generating ENTIRE ...")
     with GeneratorPerfMesurer(gen, trace_min_duration="0.5") as pm:
+        if not gen.supports_device(device):
+            logger.debug("generator %s is not supported for device %s", gen, device.hostname)
+            return
+
+        path = gen.path(device)
+        if not path:
+            raise RuntimeError("entire generator should return non-empty path")
+
+        logger.info("Generating ENTIRE ...")
+
         output = gen(device)
 
     return GeneratorEntireResult(
@@ -329,35 +332,37 @@ def _make_generator_ctx(gen):
 
 
 def _run_json_fragment_generator(
-        gen: "JSONFragment",
-        device: "Device",
+    gen: "JSONFragment",
+    device: "Device",
 ) -> Optional[GeneratorResult]:
     logger = get_logger(generator=_make_generator_ctx(gen))
-    if not gen.supports_device(device):
-        logger.info("generator %s is not supported for device %s", gen, device.hostname)
-        return
-
-    path = gen.path(device)
-    if not path:
-        raise RuntimeError("json fragment generator should return non-empty path")
-
-    acl_item_or_list_of_items = gen.acl(device)
-    safe_acl_item_or_list_of_items = gen.acl_safe(device)
-    if not acl_item_or_list_of_items:
-        raise RuntimeError("json fragment generator should return non-empty acl")
-    if isinstance(acl_item_or_list_of_items, list):
-        acl = acl_item_or_list_of_items
-    else:
-        acl = [acl_item_or_list_of_items]
-    if isinstance(safe_acl_item_or_list_of_items, list):
-        acl_safe = safe_acl_item_or_list_of_items
-    else:
-        acl_safe = [safe_acl_item_or_list_of_items]
-
-    logger.info("Generating JSON_FRAGMENT ...")
+
     with GeneratorPerfMesurer(gen) as pm:
+        if not gen.supports_device(device):
+            logger.info("generator %s is not supported for device %s", gen, device.hostname)
+            return
+
+        path = gen.path(device)
+        if not path:
+            raise RuntimeError("json fragment generator should return non-empty path")
+
+        acl_item_or_list_of_items = gen.acl(device)
+        safe_acl_item_or_list_of_items = gen.acl_safe(device)
+        if not acl_item_or_list_of_items:
+            raise RuntimeError("json fragment generator should return non-empty acl")
+        if isinstance(acl_item_or_list_of_items, list):
+            acl = acl_item_or_list_of_items
+        else:
+            acl = [acl_item_or_list_of_items]
+        if isinstance(safe_acl_item_or_list_of_items, list):
+            acl_safe = safe_acl_item_or_list_of_items
+        else:
+            acl_safe = [safe_acl_item_or_list_of_items]
+
+        logger.info("Generating JSON_FRAGMENT ...")
+
         config = gen(device)
-    reload_cmds = gen.get_reload_cmds(device)
+        reload_cmds = gen.get_reload_cmds(device)
     return GeneratorJSONFragmentResult(
         name=gen.__class__.__name__,
         tags=gen.TAGS,
@@ -407,7 +412,8 @@ def _load_gen_module(module_path: str):
     except ModuleNotFoundError as e:
         try:  # maybe it's a path to module
             module_abs_path = os.path.abspath(module_path)
-            module = importlib.machinery.SourceFileLoader(re.sub(r"[./]", "_", module_abs_path).strip("_"), module_abs_path).load_module()
+            module = importlib.machinery.SourceFileLoader(re.sub(r"[./]", "_", module_abs_path).strip("_"),
+                                                          module_abs_path).load_module()
         except ModuleNotFoundError:
             raise e
     return module

{annet-3.2.0 → annet-3.4.0}/annet/generators/entire.py

@@ -3,19 +3,10 @@ from __future__ import annotations
 import pkgutil
 import re
 import types
-from typing import (
-    FrozenSet,
-    Iterable,
-    List,
-    Optional,
-    Set,
-    Union,
-)
-
-from annet.lib import (
-    flatten,
-    mako_render,
-)
+from typing import FrozenSet, Iterable, List, Optional, Set, Union
+
+from annet.lib import flatten, mako_render
+
 from .base import BaseGenerator, _filter_str
 from .exceptions import NotSupportedDevice
 
@@ -39,8 +30,7 @@ class Entire(BaseGenerator):
     def run(self, device) -> Union[None, str, Iterable[Union[str, tuple]]]:
         raise NotImplementedError
 
-    def reload(self, device) -> Optional[
-        str]:  # pylint: disable=unused-argument
+    def reload(self, device) -> Optional[str]:  # pylint: disable=unused-argument
         return
 
     def get_reload_cmds(self, device) -> str:

{annet-3.2.0 → annet-3.4.0}/annet/generators/perf.py

@@ -1,27 +1,19 @@
-from __future__ import annotations
-
 import time
-from typing import (
-    Optional,
-    Union,
-)
+from typing import Optional, Union
 
 from annet import tracing
 from annet.tracing import tracing_connector
-from annet.types import (
-    GeneratorPartialRunArgs,
-    GeneratorPerf,
-)
-from .entire import Entire
-from .partial import PartialGenerator
+from annet.types import GeneratorPartialRunArgs, GeneratorPerf
+
+from .base import BaseGenerator
 
 
 class GeneratorPerfMesurer:
     def __init__(
-            self,
-            gen: Union[PartialGenerator, Entire],
-            run_args: Optional[GeneratorPartialRunArgs] = None,
-            trace_min_duration: tracing.MinDurationT = None
+        self,
+        gen: BaseGenerator,
+        run_args: Optional[GeneratorPartialRunArgs] = None,
+        trace_min_duration: tracing.MinDurationT = None
     ):
         self._gen = gen
         self._run_args = run_args
@@ -44,8 +36,7 @@ class GeneratorPerfMesurer:
         self._span = self._span_ctx.__enter__()  # pylint: disable=unnecessary-dunder-call
 
         if self._span:
-            self._span.set_attributes(
-                {"generator.class": self._gen.__class__.__name__})
+            self._span.set_attributes({"generator.class": self._gen.__class__.__name__})
             if self._run_args:
                 tracing_connector.get().set_device_attributes(
                     self._span, self._run_args.device,

{annet-3.2.0 → annet-3.4.0/annet.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.2.0
+Version: 3.4.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -21,7 +21,7 @@ Requires-Dist: yarl>=1.8.2
 Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.3.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.4.0; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: license-file

{annet-3.2.0 → annet-3.4.0}/annet.egg-info/requires.txt

@@ -13,4 +13,4 @@ adaptix==3.0.0b7
 dataclass-rest==0.4
 
 [netbox]
-annetbox[sync]>=0.3.0
+annetbox[sync]>=0.4.0

{annet-3.2.0 → annet-3.4.0}/setup.py

@@ -44,7 +44,7 @@ if __name__ == "__main__":
           ],
         },
         extras_require={
-            "netbox": ["annetbox[sync]>=0.3.0"],
+            "netbox": ["annetbox[sync]>=0.4.0"],
         },
         python_requires=">=3.10",
         install_requires=requirements(),