annet 3.18.0__tar.gz → 3.20.0__tar.gz

{annet-3.18.0/annet.egg-info → annet-3.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.18.0
+Version: 3.20.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -22,7 +22,8 @@ Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Requires-Dist: ordered-set>=4.1.0
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.8.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.9.0; extra == "netbox"
+Requires-Dist: requests-cache>=1.2.1; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: license-file

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/common/storage_base.py

@@ -1,7 +1,11 @@
 import ssl
 from ipaddress import ip_interface
 from logging import getLogger
-from typing import Any, Optional, List, Union, Dict, cast, Generic, TypeVar
+from typing import Any, Callable, Optional, List, Union, Dict, cast, Generic, TypeVar
+
+from requests import Session
+
+from requests_cache import CachedSession, SQLiteCache
 
 from annetbox.v37 import models as api_models
 
@@ -45,6 +49,7 @@ class BaseNetboxStorage(
         token = ""
         self.exact_host_filter = False
         threads = 1
+        session_factory = None
         if opts:
             if opts.insecure:
                 ctx = ssl.create_default_context()
@@ -55,7 +60,17 @@ class BaseNetboxStorage(
             threads = opts.threads
             self.exact_host_filter = opts.exact_host_filter
             self.all_hosts_filter = opts.all_hosts_filter
-        self.netbox = self._init_adapter(url=url, token=token, ssl_context=ctx, threads=threads)
+
+            if opts.cache_path:
+                session_factory = cached_requests_session(opts)
+
+        self.netbox = self._init_adapter(
+            url=url,
+            token=token,
+            ssl_context=ctx,
+            threads=threads,
+            session_factory=session_factory,
+        )
         self._all_fqdns: Optional[list[str]] = None
         self._id_devices: dict[int, NetboxDeviceT] = {}
         self._name_devices: dict[str, NetboxDeviceT] = {}
@@ -67,6 +82,7 @@ class BaseNetboxStorage(
             token: str,
             ssl_context: Optional[ssl.SSLContext],
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT, FHRPGroupT, FHRPGroupAssignmentT]:
         raise NotImplementedError()
 
@@ -259,3 +275,16 @@ def parse_glob(exact_host_filter: bool, query: NetboxQuery) -> dict[str, list[st
         else:
             query_groups["name__ic"] = [_hostname_dot_hack(name) for name in names]
     return query_groups
+
+
+def cached_requests_session(opts: NetboxStorageOpts) -> Callable[[Session], Session]:
+    def session_factory(session: Session) -> Session:
+        backend = SQLiteCache(db_path=opts.cache_path)
+        if opts.recache:
+            backend.clear()
+        return CachedSession.wrap(
+            session,
+            backend=backend,
+            expire_after=opts.cache_ttl,
+        )
+    return session_factory

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/common/storage_opts.py

@@ -1,5 +1,6 @@
 import os
 from typing import Any, Optional
+from datetime import timedelta
 from .query import parse_query
 
 DEFAULT_URL = "http://localhost"
@@ -14,6 +15,9 @@ class NetboxStorageOpts:
             exact_host_filter: bool = False,
             threads: int = 1,
             all_hosts_filter: dict[str, list[str]] | None = None,
+            cache_path: str = "",
+            cache_ttl: timedelta = timedelta(0),
+            recache: bool = False,
     ):
         self.url = url
         self.token = token
@@ -21,6 +25,9 @@ class NetboxStorageOpts:
         self.exact_host_filter = exact_host_filter
         self.threads = threads
         self.all_hosts_filter: dict[str, list[str]] = all_hosts_filter or {}
+        self.cache_path = cache_path
+        self.cache_ttl = cache_ttl
+        self.recache = recache
 
     @classmethod
     def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
@@ -41,6 +48,17 @@ class NetboxStorageOpts:
             exact_host_filter = exact_host_filter_env in ("true", "1", "t")
         else:
             exact_host_filter = bool(conf_params.get("exact_host_filter") or False)
+
+        if cache_path := os.getenv("NETBOX_CACHE_PATH"):
+            cache_path = cache_path
+        else:
+            cache_path = str(conf_params.get("cache_path", ""))
+
+        if cache_ttl := os.getenv("NETBOX_CACHE_TTL"):
+            cache_ttl = timedelta(seconds=int(cache_ttl))
+        else:
+            cache_ttl = timedelta(seconds=conf_params.get("cache_ttl", 0))
+
         return cls(
             url=url,
             token=token,
@@ -48,4 +66,7 @@ class NetboxStorageOpts:
             exact_host_filter=exact_host_filter,
             threads=int(threads),
             all_hosts_filter=all_hosts_filter,
+            cache_path=cache_path,
+            cache_ttl=cache_ttl,
+            recache=cli_opts.recache,
         )

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/provider.py

@@ -46,7 +46,8 @@ def storage_factory(opts: NetboxStorageOpts) -> Storage:
 
 class NetboxProvider(StorageProvider, AdapterWithName, AdapterWithConfig):
     def __init__(self, url: Optional[str] = None, token: Optional[str] = None, insecure: bool = False,
-                 exact_host_filter: bool = False, threads: int = 1, all_hosts_filter: dict[str, list[str]] | None = None):
+                 exact_host_filter: bool = False, threads: int = 1, all_hosts_filter: dict[str, list[str]] | None = None,
+                 cache_path: str = "", cache_ttl: int = 0):
         self.url = url
         self.token = token
         self.insecure = insecure

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/v37/storage.py

@@ -1,4 +1,7 @@
 import ssl
+from typing import Callable
+
+from requests import Session
 
 from adaptix import P
 from adaptix.conversion import get_converter, link_function, link_constant, link
@@ -23,8 +26,9 @@ class NetboxV37Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV37(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV37(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV37,
@@ -120,5 +124,6 @@ class NetboxStorageV37(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceV37, InterfaceV37, IpAddressV37, PrefixV37, FHRPGroupV37, FHRPGroupAssignmentV37]:
-        return NetboxV37Adapter(self, url, token, ssl_context, threads)
+        return NetboxV37Adapter(self, url, token, ssl_context, threads, session_factory)

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/v41/storage.py

@@ -1,4 +1,8 @@
 import ssl
+from typing import Callable
+
+from requests import Session
+
 from adaptix import P
 from adaptix.conversion import get_converter, link, link_constant, link_function
 from annetbox.v41 import client_sync
@@ -24,8 +28,9 @@ class NetboxV41Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV41(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV41(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV41,
@@ -122,5 +127,6 @@ class NetboxStorageV41(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceV41, InterfaceV41, IpAddressV41, PrefixV41, FHRPGroupV41, FHRPGroupAssignmentV41]:
-        return NetboxV41Adapter(self, url, token, ssl_context, threads)
+        return NetboxV41Adapter(self, url, token, ssl_context, threads, session_factory)

{annet-3.18.0 → annet-3.20.0}/annet/adapters/netbox/v42/storage.py

@@ -1,5 +1,7 @@
 import ssl
-from typing import Optional
+from typing import Callable, Optional
+
+from requests import Session
 
 from adaptix import P
 from adaptix.conversion import get_converter, link, link_constant, link_function
@@ -25,8 +27,9 @@ class NetboxV42Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV42(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV42(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV42,
@@ -136,11 +139,12 @@ class NetboxStorageV42(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[
         NetboxDeviceV42, InterfaceV42, IpAddressV42, PrefixV42,
         FHRPGroupV41, FHRPGroupAssignmentV41,
     ]:
-        return NetboxV42Adapter(self, url, token, ssl_context, threads)
+        return NetboxV42Adapter(self, url, token, ssl_context, threads, session_factory)
 
     def resolve_all_vlans(self) -> list[Vlan]:
         if self._all_vlans is None:

{annet-3.18.0 → annet-3.20.0}/annet/cli_args.py

@@ -140,6 +140,12 @@ opt_tolerate_fails = Arg(
     help="Show errors without interrupting the generation"
 )
 
+opt_recache = Arg(
+    "--recache", default=False,
+    help="Force expiration of storage's local cache if it is supported"
+)
+
+
 # При параллельном запуске и включённом --tolerate-fails код возврата
 # всегда нулевой. Это не позволяет нам легко понять, прошла ли генерация
 # успешно для всех устройств. С этим флажком код будет ненулевой, если
@@ -353,6 +359,7 @@ opt_selected_context_name = Arg(
 
 # ====
 class CacheOptions(ArgGroup):
+    recache = opt_recache
     no_mesh = False
 
 

{annet-3.18.0 → annet-3.20.0}/annet/generators/__init__.py

@@ -26,11 +26,12 @@ from annet.types import (
 )
 from annet.vendors import registry_connector, tabparser
 
+from .annotate import AbstractAnnotateFormatter, annotate_formatter_connector
 from .base import BaseGenerator
 from .base import ParamsList as ParamsList
 from .base import TextGenerator as TextGenerator
 from .entire import Entire
-from .exceptions import GeneratorError, NotSupportedDevice
+from .exceptions import GeneratorError, InvalidValueFromGenerator, NotSupportedDevice
 from .jsonfragment import JSONFragment
 from .partial import PartialGenerator
 from .perf import GeneratorPerfMesurer
@@ -165,11 +166,10 @@ def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRu
     safe_config = odict()
 
     if not gen.supports_device(device):
-        logger.info("generator %s is not supported for device %s", gen, device.hostname)
+        logger.debug("generator %s is not supported for device %s", gen, device.hostname)
         return None
 
-    span = tracing_connector.get().get_current_span()
-    if span:
+    if span := tracing_connector.get().get_current_span():
         tracing_connector.get().set_device_attributes(span, run_args.device)
         tracing_connector.get().set_dimensions_attributes(span, gen, run_args.device)
         span.set_attributes({
@@ -184,8 +184,7 @@ def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRu
                 logger.info("Generating PARTIAL ...")
             try:
                 output = gen(device, run_args.annotate)
-            except NotSupportedDevice:
-                # это исключение нужно передать выше в оригинальном виде
+            except NotSupportedDevice:  # это исключение нужно передать выше в оригинальном виде
                 raise
             except Exception as err:
                 filename, lineno = gen.get_running_line()

annet-3.20.0/annet/generators/annotate.py

@@ -0,0 +1,37 @@
+import abc
+from types import GeneratorType
+
+from annet.connectors import Connector
+
+from .base import BaseGenerator
+
+
+class AbstractAnnotateFormatter(abc.ABC):
+    def __init__(self, gen: BaseGenerator):
+        self.gen = gen
+        self._annotation_module = ".".join(gen.__class__.__module__.split(".")[-2:])
+
+    @abc.abstractmethod
+    def make_annotation(self, running_gen: GeneratorType) -> str:
+        raise NotImplementedError
+
+    def get_running_line(self, running_gen: GeneratorType) -> tuple[str, int]:
+        if not running_gen or not running_gen.gi_frame:
+            return repr(running_gen), -1
+        return self._annotation_module, running_gen.gi_frame.f_lineno
+
+
+class DefaultAnnotateFormatter(AbstractAnnotateFormatter):
+    def make_annotation(self, running_gen: GeneratorType) -> str:
+        return "%s:%d" % self.get_running_line(running_gen)
+
+
+class _AnnotateFormatterConnector(Connector[AbstractAnnotateFormatter]):
+    name = "AnnotateFormatterConnector"
+    ep_name = "annotate_formatter"
+
+    def _get_default(self) -> type[AbstractAnnotateFormatter]:
+        return DefaultAnnotateFormatter
+
+
+annotate_formatter_connector = _AnnotateFormatterConnector()

{annet-3.18.0 → annet-3.20.0}/annet/generators/base.py

@@ -2,15 +2,20 @@ from __future__ import annotations
 
 import abc
 import contextlib
+import re
 import textwrap
-from typing import Union, List
+from typing import List, Union
 
 from annet import tracing
-from annet.vendors import tabparser
 from annet.tracing import tracing_connector
+from annet.vendors import tabparser
+
 from .exceptions import InvalidValueFromGenerator
 
 
+NONE_SEARCHER = re.compile(r"\bNone\b")
+
+
 class DefaultBlockIfCondition:
     pass
 
@@ -53,11 +58,15 @@ def _split_and_strip(text):
 # =====
 class BaseGenerator:
     TYPE: str
-    TAGS: List[str]
+    TAGS: List[str] = []
 
     def supports_device(self, device) -> bool:  # pylint: disable=unused-argument
         return True
 
+    @classmethod
+    def get_aliases(cls) -> set[str]:
+        return {cls.__name__, *cls.TAGS}
+
 
 class TreeGenerator(BaseGenerator):
     def __init__(self, indent="  "):

{annet-3.18.0 → annet-3.20.0}/annet/generators/entire.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
 import pkgutil
-import re
 import types
-from typing import FrozenSet, Iterable, List, Optional, Set, Union
+from typing import FrozenSet, Iterable, List, Optional, Union
 
 from annet.lib import flatten, mako_render
 
-from .base import BaseGenerator, _filter_str
-from .exceptions import NotSupportedDevice
+from .base import NONE_SEARCHER, BaseGenerator, _filter_str
+from .exceptions import InvalidValueFromGenerator
 
 
 class Entire(BaseGenerator):
@@ -62,10 +61,6 @@ class Entire(BaseGenerator):
 
     # =====
 
-    @classmethod
-    def get_aliases(cls) -> Set[str]:
-        return {cls.__name__, *cls.TAGS}
-
     def __call__(self, device):
         self.__device = device
         parts = []
@@ -73,15 +68,19 @@ class Entire(BaseGenerator):
         if isinstance(run_res, str):
             run_res = (run_res,)
         if run_res is None or not isinstance(run_res, (tuple, types.GeneratorType)):
-            raise Exception("generator %s returns %s" % (
-                self.__class__.__name__, type(run_res)))
+            raise Exception("generator %s returns %s" % (self.__class__.__name__, type(run_res)))
+
         for text in run_res:
             if isinstance(text, tuple):
                 text = " ".join(map(_filter_str, flatten(text)))
-            assert re.search(r"\bNone\b", text) is None, \
-                "Found 'None' in yield result: %s" % text
+            if NONE_SEARCHER.search(text):
+                raise InvalidValueFromGenerator(
+                    "Found 'None' in yield result: %s" % text
+                )
             parts.append(text)
+
         ret = "\n".join(parts)
         if self.ENSURE_END_NEWLINE and not ret.endswith("\n"):
             ret += "\n"
+
         return ret

{annet-3.18.0 → annet-3.20.0}/annet/generators/jsonfragment.py

@@ -1,16 +1,10 @@
 from __future__ import annotations
 
 import contextlib
-from typing import (
-    Any,
-    Dict,
-    List,
-    Optional,
-    Set,
-    Union,
-)
+from typing import Any, Optional, Union
 
 from annet.storage import Device, Storage
+
 from .base import TreeGenerator, _filter_str
 
 
@@ -18,13 +12,12 @@ class JSONFragment(TreeGenerator):
     """Generates parts of JSON config file."""
 
     TYPE = "JSON_FRAGMENT"
-    TAGS: List[str] = []
 
     def __init__(self, storage: Storage):
         super().__init__()
         self.storage = storage
-        self._json_config: Dict[str, Any] = {}
-        self._config_pointer: List[str] = []
+        self._json_config: dict[str, Any] = {}
+        self._config_pointer: list[str] = []
 
         # if two generators edit same file, commands from generator with greater `reload_prio` will be used
         if not hasattr(self, "reload_prio"):
@@ -36,11 +29,7 @@ class JSONFragment(TreeGenerator):
     def path(self, device: Device) -> Optional[str]:
         raise NotImplementedError("Required PATH for JSON_FRAGMENT generator")
 
-    @classmethod
-    def get_aliases(cls) -> Set[str]:
-        return {cls.__name__, *cls.TAGS}
-
-    def acl(self, device: Device) -> Union[str, List[str]]:
+    def acl(self, device: Device) -> Union[str, list[str]]:
         """
         Restrict the generator to a specified ACL using JSON Pointer syntax.
 
@@ -48,7 +37,7 @@ class JSONFragment(TreeGenerator):
         """
         raise NotImplementedError("Required ACL for JSON_FRAGMENT generator")
 
-    def acl_safe(self, device: Device) -> Union[str, List[str]]:
+    def acl_safe(self, device: Device) -> Union[str, list[str]]:
         """
         Restrict the generator to a specified safe ACL using JSON Pointer syntax.
 

{annet-3.18.0 → annet-3.20.0}/annet/generators/partial.py

@@ -1,32 +1,23 @@
 from __future__ import annotations
 
-import re
-from typing import (
-    Iterable,
-    List,
-    Set,
-    Union,
-)
-
-from annet.lib import (
-    add_annotation,
-    flatten,
-)
-from .base import TreeGenerator, _filter_str
-from .exceptions import NotSupportedDevice
+from typing import Iterable, Union
+
+from annet.lib import add_annotation, flatten
+
+from .annotate import AbstractAnnotateFormatter, annotate_formatter_connector
+from .base import NONE_SEARCHER, TreeGenerator, _filter_str
+from .exceptions import InvalidValueFromGenerator
 
 
 class PartialGenerator(TreeGenerator):
     TYPE = "PARTIAL"
-    TAGS: List[str] = []
 
     def __init__(self, storage):
         super().__init__()
         self.storage = storage
-        self._annotate = False
         self._running_gen = None
+        self._annotate: AbstractAnnotateFormatter = annotate_formatter_connector.get(self)
         self._annotations = []
-        self._annotation_module = self.__class__.__module__ or ""
 
     def supports_device(self, device) -> bool:
         if self.__class__.run is PartialGenerator.run:
@@ -35,19 +26,19 @@ class PartialGenerator(TreeGenerator):
             return True
 
     def acl(self, device):
-        acl_func = self._get_vendor_func(device.hw.vendor, "acl")
-        if acl_func:
+        if acl_func := self._get_vendor_func(device.hw.vendor, "acl"):
             return acl_func(device)
+        return None
 
     def acl_safe(self, device):
-        acl_func = self._get_vendor_func(device.hw.vendor, "acl_safe")
-        if acl_func:
+        if acl_func := self._get_vendor_func(device.hw.vendor, "acl_safe"):
             return acl_func(device)
+        return None
 
-    def run(self, device) -> Iterable[Union[str, tuple]]:
-        run_func = self._get_vendor_func(device.hw.vendor, "run")
-        if run_func:
+    def run(self, device) -> Iterable[Union[str, tuple]] | None:
+        if run_func := self._get_vendor_func(device.hw.vendor, "run"):
             return run_func(device)
+        return None
 
     def get_user_runner(self, device):
         if self.__class__.run is not PartialGenerator.run:
@@ -56,26 +47,15 @@ class PartialGenerator(TreeGenerator):
 
     def _get_vendor_func(self, vendor: str, func_name: str):
         attr_name = f"{func_name}_{vendor}"
-        if hasattr(self, attr_name):
-            return getattr(self, attr_name)
-        return None
+        return getattr(self, attr_name, None)
 
     # =====
 
-    @classmethod
-    def get_aliases(cls) -> Set[str]:
-        return {cls.__name__, *cls.TAGS}
-
-    def __call__(self, device, annotate=False):
+    def __call__(self, device, annotate: bool = False):
         self._indents = []
         self._rows = []
-        self._running_gen = self.run(device)
-        self._annotate = annotate
-
-        if annotate and self.__class__.__module__:
-            self._annotation_module = ".".join(
-                self.__class__.__module__.split(".")[-2:])
 
+        self._running_gen = self.run(device)
         for text in self._running_gen:
             if isinstance(text, tuple):
                 text = " ".join(map(_filter_str, flatten(text)))
@@ -83,9 +63,12 @@ class PartialGenerator(TreeGenerator):
                 text = _filter_str(text)
             self._append_text(text)
 
-        for row in self._rows:
-            assert re.search(r"\bNone\b", row) is None, \
-                "Found 'None' in yield result: %s" % (row)
+        for (row, annotation) in zip(self._rows, self._annotations):
+            if NONE_SEARCHER.search(row):
+                raise InvalidValueFromGenerator(
+                    "Found 'None' in yield result: %s" % add_annotation(row, annotation)
+                )
+
         if annotate:
             generated_rows = (
                 add_annotation(x, y)
@@ -93,23 +76,20 @@ class PartialGenerator(TreeGenerator):
             )
         else:
             generated_rows = self._rows
-        return "\n".join(generated_rows) + "\n"
+
+        return "\n".join((*generated_rows, ""))
 
     def _append_text(self, text):
         def annotation_cb(row):
-            annotation = "%s:%d" % self.get_running_line()
-            self._annotations.append(annotation)
+            self._annotations.append(
+                self._annotate.make_annotation(self._running_gen)
+            )
             return row
 
-        self._append_text_cb(
-            text,
-            annotation_cb if self._annotate else None
-        )
+        self._append_text_cb(text, row_cb=annotation_cb)
 
-    def get_running_line(self):
-        if not self._running_gen or not self._running_gen.gi_frame:
-            return (repr(self._running_gen), -1)
-        return self._annotation_module, self._running_gen.gi_frame.f_lineno
+    def get_running_line(self) -> tuple[str, int]:
+        return self._annotate.get_running_line(self._running_gen)
 
     @classmethod
     def literal(cls, item):