annet 3.10.0__tar.gz → 3.12.0__tar.gz

{annet-3.10.0/annet.egg-info → annet-3.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.10.0
+Version: 3.12.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/common/adapter.py

@@ -47,7 +47,7 @@ class NetboxAdapter(
     ],
 ):
     @abstractmethod
-    def list_all_fqdns(self) -> list[str]:
+    def list_fqdns(self, query: dict[str, list[str]] | None = None) -> list[str]:
         raise NotImplementedError()
 
     @abstractmethod

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/common/query.py

@@ -37,19 +37,7 @@ class NetboxQuery(Query):
         return self.query
 
     def parse_query(self) -> Filter:
-        query_groups = defaultdict(list)
-        for q in self.globs:
-            if FIELD_VALUE_SEPARATOR in q:
-                glob_type, param = q.split(FIELD_VALUE_SEPARATOR, 2)
-                if glob_type not in ALLOWED_GLOB_GROUPS:
-                    raise Exception(f"unknown query type: '{glob_type}'")
-                if not param:
-                    raise Exception(f"empty param for '{glob_type}'")
-                query_groups[glob_type].append(param)
-            else:
-                query_groups["name"].append(q)
-
-        query_groups.default_factory = None
+        query_groups = parse_query(self.globs)
         return cast(Filter, query_groups)
 
     def is_empty(self) -> bool:
@@ -62,3 +50,18 @@ class NetboxQuery(Query):
             if FIELD_VALUE_SEPARATOR in q:
                 return False
         return True
+
+
+def parse_query(query: list[str]) -> dict[str, list[str]]:
+    query_groups = defaultdict(list)
+    for q in query:
+        if FIELD_VALUE_SEPARATOR in q:
+            glob_type, param = q.split(FIELD_VALUE_SEPARATOR, 2)
+            if glob_type not in ALLOWED_GLOB_GROUPS:
+                raise Exception(f"unknown query type: '{glob_type}'")
+            if not param:
+                raise Exception(f"empty param for '{glob_type}'")
+            query_groups[glob_type].append(param)
+        else:
+            query_groups["name"].append(q)
+    return dict(query_groups)

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/common/storage_base.py

@@ -54,6 +54,7 @@ class BaseNetboxStorage(
             token = opts.token
             threads = opts.threads
             self.exact_host_filter = opts.exact_host_filter
+            self.all_hosts_filter = opts.all_hosts_filter
         self.netbox = self._init_adapter(url=url, token=token, ssl_context=ctx, threads=threads)
         self._all_fqdns: Optional[list[str]] = None
         self._id_devices: dict[int, NetboxDeviceT] = {}
@@ -87,7 +88,7 @@ class BaseNetboxStorage(
 
     def resolve_all_fdnds(self) -> list[str]:
         if self._all_fqdns is None:
-            self._all_fqdns = self.netbox.list_all_fqdns()
+            self._all_fqdns = self.netbox.list_fqdns(self.all_hosts_filter)
         return self._all_fqdns
 
     def make_devices(

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/common/storage_opts.py

@@ -1,5 +1,6 @@
 import os
 from typing import Any, Optional
+from .query import parse_query
 
 DEFAULT_URL = "http://localhost"
 
@@ -12,17 +13,24 @@ class NetboxStorageOpts:
             insecure: bool = False,
             exact_host_filter: bool = False,
             threads: int = 1,
+            all_hosts_filter: dict[str, list[str]] | None = None,
     ):
         self.url = url
         self.token = token
         self.insecure = insecure
         self.exact_host_filter = exact_host_filter
         self.threads = threads
+        self.all_hosts_filter: dict[str, list[str]] = all_hosts_filter or {}
 
     @classmethod
     def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
         url = os.getenv("NETBOX_URL") or conf_params.get("url") or DEFAULT_URL
         token = os.getenv("NETBOX_TOKEN", "").strip() or conf_params.get("token") or ""
+        all_hosts_filter = None
+        if all_hosts_filter_env := os.getenv("NETBOX_ALL_HOSTS_FILTER", "").strip():
+            all_hosts_filter = parse_query(all_hosts_filter_env.split(","))
+        elif all_hosts_filter_params := conf_params.get("all_hosts_filter"):
+            all_hosts_filter = all_hosts_filter_params
         threads = os.getenv("NETBOX_CLIENT_THREADS", "").strip() or conf_params.get("threads") or "1"
         insecure = False
         if insecure_env := os.getenv("NETBOX_INSECURE", "").lower():
@@ -39,4 +47,5 @@ class NetboxStorageOpts:
             insecure=insecure,
             exact_host_filter=exact_host_filter,
             threads=int(threads),
+            all_hosts_filter=all_hosts_filter,
         )

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/v37/storage.py

@@ -70,10 +70,11 @@ class NetboxV37Adapter(NetboxAdapter[
             list[FHRPGroupV37],
         )
 
-    def list_all_fqdns(self) -> list[str]:
+    def list_fqdns(self, query: dict[str, list[str]] | None = None) -> list[str]:
+        query = query or {}
         return [
             d.name
-            for d in self.netbox.dcim_all_devices_brief().results
+            for d in self.netbox.dcim_all_devices_brief(**query).results
         ]
 
     def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceV37]:

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/v41/storage.py

@@ -71,10 +71,11 @@ class NetboxV41Adapter(NetboxAdapter[
             list[FHRPGroupV41],
         )
 
-    def list_all_fqdns(self) -> list[str]:
+    def list_fqdns(self, query: dict[str, list[str]] | None = None) -> list[str]:
+        query = query or {}
         return [
             d.name
-            for d in self.netbox.dcim_all_devices_brief().results
+            for d in self.netbox.dcim_all_devices_brief(**query).results
         ]
 
     def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceV41]:

{annet-3.10.0 → annet-3.12.0}/annet/adapters/netbox/v42/storage.py

@@ -72,10 +72,11 @@ class NetboxV42Adapter(NetboxAdapter[
             list[FHRPGroupV41],
         )
 
-    def list_all_fqdns(self) -> list[str]:
+    def list_fqdns(self, query: dict[str, list[str]] | None = None) -> list[str]:
+        query = query or {}
         return [
             d.name
-            for d in self.netbox.dcim_all_devices_brief().results
+            for d in self.netbox.dcim_all_devices_brief(**query).results
         ]
 
     def list_devices(self, query: dict[str, list[str]]) -> list[NetboxDeviceV42]:

{annet-3.10.0 → annet-3.12.0}/annet/annlib/netdev/devdb/__init__.py

@@ -1,10 +1,11 @@
 import functools
 import json
 import re
-from os import path
+from pathlib import Path
 from typing import Any, Dict
 
 from annet.annlib.netdev.db import find_true_sequences, get_db
+from annet.lib import get_context
 
 
 @functools.lru_cache(None)
@@ -23,6 +24,7 @@ def _prepare_db() -> Dict[str, Any]:
         from library.python import resource
         raw = json.loads(resource.resfs_read("contrib/python/annet/annet/annlib/netdev/devdb/data/devdb.json").decode("utf-8"))
     except ImportError:
-        with open(path.join(path.dirname(__file__), "data", "devdb.json"), "r") as f:
+        devdb_file = Path(get_context().get("devdb", {}).get("path", Path(__file__).parent / "data" / "devdb.json"))
+        with devdb_file.open("r", encoding="utf-8") as f:
             raw = json.load(f)
     return {tuple(seq.split(".")): re.compile(regexp) for (seq, regexp) in raw.items()}

{annet-3.10.0 → annet-3.12.0}/annet/api/__init__.py

@@ -331,7 +331,15 @@ def diff(
     loader: ann_gen.Loader,
     device_ids: List[Any]
 ) -> tuple[Mapping[Device, Union[Diff, PCDiff]], Mapping[Device, Exception]]:
-    """ Сгенерировать конфиг для устройств """
+    """ Сгенерировать дифф для устройств """
+    if args.config == "running":
+        fetcher = annet.deploy.get_fetcher()
+        ann_gen.live_configs = annet.lib.do_async(
+            fetcher.fetch(
+                [device for device in loader.devices if device.id in device_ids],
+                processes=args.parallel
+            )
+        )
     stdin = args.stdin(filter_acl=args.filter_acl, config=None)
 
     filterer = filtering.filterer_connector.get()

{annet-3.10.0 → annet-3.12.0}/annet/rulebook/aruba/ap_env.py

@@ -1,7 +1,7 @@
 # pylint: disable=unused-argument
 
 from annet.annlib.types import Op
-from annet.executor import CommandList, Command
+from annet.annlib.command import Command, CommandList
 
 
 def apply(hw, do_commit, do_finalize, **_):

{annet-3.10.0 → annet-3.12.0}/annet/rulebook/texts/huawei.rul

@@ -200,9 +200,11 @@ aaa
 
 hwtacacs-server template *
     hwtacacs-server shared-key
+    hwtacacs-server source-ip
 
 hwtacacs server template *
     hwtacacs server shared-key
+    hwtacacs server source-ip
 
 !radius-server template default
 

{annet-3.10.0 → annet-3.12.0}/annet/vendors/tabparser.py

@@ -64,8 +64,36 @@ class FormatterContext:
         return self.next and self.next[0]
 
 
+class NotUniquePatch:
+    def __init__(self):
+        """In the case of comments, odict is not suitable: there may be several identical edit and exit"""
+        self._items = []
+        self._keys = set()
+
+    def __setitem__(self, key, value):
+        self._keys.add(key)
+        self._items.append((key, value))
+
+    def keys(self):
+        return list(self)
+
+    def items(self):
+        return self._items
+
+    def __contains__(self, item):
+        return item in self._keys
+
+    def __iter__(self):
+        return iter(item[0] for item in self._items)
+
+    def __bool__(self) -> bool:
+        return bool(self._items)
+
+
 # =====
 class CommonFormatter:
+    cmd_path_cls = odict
+
     def __init__(self, indent="  "):
         self._indent = indent
         self._block_begin = ""
@@ -96,7 +124,7 @@ class CommonFormatter:
         )
 
     def cmd_paths(self, patch: "PatchTree") -> odict:
-        ret = odict()
+        ret = self.cmd_path_cls()
         path = []
         for row, context in self.blocks_and_context(patch, is_patch=True):
             if row is BlockBegin:
@@ -222,7 +250,20 @@ class BlockExitFormatter(CommonFormatter):
                     yield exit_statement, last_row_context
 
 
+class HuaweiPatch(NotUniquePatch):
+    policy_end_blocks = ("end-list", "endif", "end-filter")
+
+    def __setitem__(self, key, value):
+        if not key:
+            return
+
+        if key not in self or (key[0].startswith("xpl") and key[-1] in self.policy_end_blocks):
+            super().__setitem__(key, value)
+
+
 class HuaweiFormatter(BlockExitFormatter):
+    cmd_path_cls = HuaweiPatch
+
     def __init__(self, indent="  "):
         super().__init__(
             block_exit="quit",
@@ -237,9 +278,8 @@ class HuaweiFormatter(BlockExitFormatter):
     def split(self, text):
         # на старых прошивка наблюдается баг с двумя пробелами в этом месте в конфиге
         # например на VRP V100R006C00SPC500 + V100R006SPH003
-        policy_end_blocks = ("end-list", "endif", "end-filter")
         tree = self.split_remove_spaces(text)
-        tree[:] = filter(lambda x: not str(x).strip().startswith(policy_end_blocks), tree)
+        tree[:] = filter(lambda x: not str(x).strip().startswith(HuaweiPatch.policy_end_blocks), tree)
         return tree
 
     def block_exit(self, context: Optional[FormatterContext]):
@@ -256,7 +296,9 @@ class HuaweiFormatter(BlockExitFormatter):
             return
 
         if parent_row.startswith("xpl route-filter"):
-            if (row.startswith(("if", "elseif")) and row.endswith("then")) and not row_next:
+            if (row.startswith(("if", "elseif")) and row.endswith("then")) and (
+                not row_next or not row_next.startswith(("elseif", "else"))
+            ):
                 yield "endif"
             elif row == "else":
                 yield "endif"
@@ -407,29 +449,9 @@ class AsrFormatter(BlockExitFormatter):
             yield from super().block_exit(context)
 
 
-class JuniperPatch:
-    def __init__(self):
-        """In the case of comments, odict is not suitable: there may be several identical edit and exit"""
-        self._items = []
-
-    def __setitem__(self, key, value):
-        self._items.append((key, value))
-
-    def keys(self):
-        return list(self)
-
-    def items(self):
-        return self._items
-
-    def __iter__(self):
-        return iter(item[0] for item in self._items)
-
-    def __bool__(self) -> bool:
-        return bool(self._items)
-
-
 class JuniperFormatter(CommonFormatter):
     patch_set_prefix = "set"
+    cmd_path_cls = NotUniquePatch
 
     @dataclasses.dataclass
     class Comment:
@@ -526,7 +548,7 @@ class JuniperFormatter(CommonFormatter):
             yield line + self._statement_end
 
     def cmd_paths(self, patch, _prev=tuple()):
-        commands = JuniperPatch()
+        commands = self.cmd_path_cls()
 
         for item in patch.itms:
             key, childs, context = item.row, item.child, item.context

{annet-3.10.0 → annet-3.12.0/annet.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.10.0
+Version: 3.12.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

{annet-3.10.0 → annet-3.12.0}/annet.egg-info/SOURCES.txt

@@ -14,7 +14,6 @@ annet/connectors.py
 annet/deploy.py
 annet/deploy_ui.py
 annet/diff.py
-annet/executor.py
 annet/filtering.py
 annet/gen.py
 annet/hardware.py

annet-3.10.0/annet/executor.py

@@ -1,172 +0,0 @@
-import asyncio
-import os
-import statistics
-from abc import ABC, abstractmethod
-from functools import partial
-from operator import itemgetter
-from typing import Any, List, Optional
-
-import colorama
-from annet.annlib.command import Command, CommandList, Question  # noqa: F401
-
-
-class CommandResult(ABC):
-    @abstractmethod
-    def get_out(self) -> str:
-        pass
-
-
-class ExecutorException(Exception):
-    def __init__(self, *args: List[Any], auxiliary: Optional[Any] = None, **kwargs: object):
-        self.auxiliary = auxiliary
-        super().__init__(*args, **kwargs)
-
-    def __repr__(self) -> str:
-        return "%s(args=%r,auxiliary=%s)" % (self.__class__.__name__, self.args, self.auxiliary)
-
-
-class ExecException(ExecutorException):
-    def __init__(self, msg: str, cmd: str, res: str, **kwargs):
-        super().__init__(**kwargs)
-        self.args = msg, cmd, res
-        self.kwargs = kwargs
-        self.msg = msg
-        self.cmd = cmd
-        self.res = res
-
-    def __str__(self) -> str:
-        return str(self.msg)
-
-    def __repr__(self) -> str:
-        return "%s<%s, %s>" % (self.__class__.__name__, self.msg, self.cmd)
-
-
-class BadCommand(ExecException):
-    pass
-
-
-class NonzeroRetcode(ExecException):
-    pass
-
-
-class CommitException(ExecException):
-    pass
-
-
-def _show_type_summary(caption, items, total, stat_items=None):
-    if items:
-        if not stat_items:
-            stat = ""
-        else:
-            avg = statistics.mean(stat_items)
-            stat = "   %(min).1f/%(max).1f/%(avg).1f/%(stdev)s  (min/max/avg/stdev)" % dict(
-                min=min(stat_items),
-                max=max(stat_items),
-                avg=avg,
-                stdev="-" if len(stat_items) < 2 else "%.1f" % statistics.stdev(stat_items, xbar=avg)
-            )
-
-        print("%-8s %d of %d%s" % (caption, len(items), total, stat))
-
-
-def show_bulk_report(hostnames, res, durations, log_dir):
-    total = len(hostnames)
-    if not total:
-        return
-
-    colorama.init()
-
-    print("\n====== bulk deploy report ======")
-
-    done = [host for (host, hres) in res.items() if not isinstance(hres, Exception)]
-    cancelled = [host for (host, hres) in res.items() if isinstance(hres, asyncio.CancelledError)]
-    failed = [host for (host, hres) in res.items() if isinstance(hres, Exception) and host not in cancelled]
-    lost = [host for host in hostnames if host not in res]
-    limit = 30
-
-    _show_type_summary("Done :", done, total, [durations[h] for h in done])
-    _print_limit(done, partial(_print_hostname, style=colorama.Fore.GREEN), limit, total)
-
-    _show_type_summary("Failed :", failed, total, [durations[h] for h in failed])
-
-    _print_limit(failed, partial(_print_failed, res=res), limit, total)
-
-    _show_type_summary("Cancelled :", cancelled, total, [durations[h] for h in cancelled if durations[h] is not None])
-    _print_limit(cancelled, partial(_print_hostname, style=colorama.Fore.RED), limit, total)
-
-    _show_type_summary("Lost :", lost, total)
-    _print_limit(lost, _print_hostname, limit, total)
-
-    err_limit = 5
-    if failed:
-        errs = {}
-        for hostname in failed:
-            fmt_err = _format_exc(res[hostname])
-            if fmt_err in errs:
-                errs[fmt_err] += 1
-            else:
-                errs[fmt_err] = 1
-        print("Top errors :")
-        for fmt_err, n in sorted(errs.items(), key=itemgetter(1), reverse=True)[:err_limit]:
-            print("  %-4d %s" % (n, fmt_err))
-        print("\n", end="")
-
-    if log_dir:
-        print("See deploy logs in %s/\n" % os.path.relpath(log_dir))
-
-
-def _format_exc(exc):
-    if isinstance(exc, ExecException):
-        cmd = str(exc.cmd)
-        if len(cmd) > 50:
-            cmd = cmd[:50] + "~.."
-        return "'%s', cmd '%s'" % (exc.msg, cmd)
-    elif isinstance(exc, ExecutorException):
-        return "%s%r" % (exc.__class__.__name__, exc.args)  # исключить многословный auxiliary
-    else:
-        return repr(exc)
-
-
-def _print_hostname(host, style=None):
-    if style:
-        host = style + host + colorama.Style.RESET_ALL
-    print("  %s" % host)
-
-
-def _print_limit(items, printer, limit, total, end="\n"):
-    if not items:
-        return
-    if len(items) > limit and len(items) > total * 0.7:
-        print("  ... %d hosts" % len(items))
-    for host in items[:limit]:
-        printer(host)
-    if len(items) > limit:
-        print("  ... %d more hosts" % (len(items) - limit))
-
-    print(end, end="")
-
-
-def _print_failed(host, res):
-    exc = res[host]
-    color = colorama.Fore.YELLOW if isinstance(exc, Warning) else colorama.Fore.RED
-    print("  %s - %s" % (color + host + colorama.Style.RESET_ALL, _format_exc(exc)))
-
-
-class DeferredFileWrite:
-    def __init__(self, file, mode="r"):
-        self._file = file
-        wrapper = {"w": "a", "wb": "ab"}
-        if mode in wrapper:
-            self._mode = wrapper[mode]
-        else:
-            raise Exception()
-
-    def write(self, data):
-        with open(self._file, self._mode, encoding="utf-8") as fh:
-            fh.write(data)
-
-    def close(self):
-        pass
-
-    def flush(self):
-        pass