annet 3.17.1__tar.gz → 3.19.0__tar.gz

{annet-3.17.1/annet.egg-info → annet-3.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.17.1
+Version: 3.19.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -22,7 +22,8 @@ Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Requires-Dist: ordered-set>=4.1.0
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.8.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.9.0; extra == "netbox"
+Requires-Dist: requests-cache>=1.2.1; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: license-file

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/common/storage_base.py

@@ -1,7 +1,11 @@
 import ssl
 from ipaddress import ip_interface
 from logging import getLogger
-from typing import Any, Optional, List, Union, Dict, cast, Generic, TypeVar
+from typing import Any, Callable, Optional, List, Union, Dict, cast, Generic, TypeVar
+
+from requests import Session
+
+from requests_cache import CachedSession, SQLiteCache
 
 from annetbox.v37 import models as api_models
 
@@ -45,6 +49,7 @@ class BaseNetboxStorage(
         token = ""
         self.exact_host_filter = False
         threads = 1
+        session_factory = None
         if opts:
             if opts.insecure:
                 ctx = ssl.create_default_context()
@@ -55,7 +60,17 @@ class BaseNetboxStorage(
             threads = opts.threads
             self.exact_host_filter = opts.exact_host_filter
             self.all_hosts_filter = opts.all_hosts_filter
-        self.netbox = self._init_adapter(url=url, token=token, ssl_context=ctx, threads=threads)
+
+            if opts.cache_path:
+                session_factory = cached_requests_session(opts)
+
+        self.netbox = self._init_adapter(
+            url=url,
+            token=token,
+            ssl_context=ctx,
+            threads=threads,
+            session_factory=session_factory,
+        )
         self._all_fqdns: Optional[list[str]] = None
         self._id_devices: dict[int, NetboxDeviceT] = {}
         self._name_devices: dict[str, NetboxDeviceT] = {}
@@ -67,6 +82,7 @@ class BaseNetboxStorage(
             token: str,
             ssl_context: Optional[ssl.SSLContext],
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceT, InterfaceT, IpAddressT, PrefixT, FHRPGroupT, FHRPGroupAssignmentT]:
         raise NotImplementedError()
 
@@ -259,3 +275,16 @@ def parse_glob(exact_host_filter: bool, query: NetboxQuery) -> dict[str, list[st
         else:
             query_groups["name__ic"] = [_hostname_dot_hack(name) for name in names]
     return query_groups
+
+
+def cached_requests_session(opts: NetboxStorageOpts) -> Callable[[Session], Session]:
+    def session_factory(session: Session) -> Session:
+        backend = SQLiteCache(db_path=opts.cache_path)
+        if opts.recache:
+            backend.clear()
+        return CachedSession.wrap(
+            session,
+            backend=backend,
+            expire_after=opts.cache_ttl,
+        )
+    return session_factory

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/common/storage_opts.py

@@ -1,5 +1,6 @@
 import os
 from typing import Any, Optional
+from datetime import timedelta
 from .query import parse_query
 
 DEFAULT_URL = "http://localhost"
@@ -14,6 +15,9 @@ class NetboxStorageOpts:
             exact_host_filter: bool = False,
             threads: int = 1,
             all_hosts_filter: dict[str, list[str]] | None = None,
+            cache_path: str = "",
+            cache_ttl: timedelta = timedelta(0),
+            recache: bool = False,
     ):
         self.url = url
         self.token = token
@@ -21,6 +25,9 @@ class NetboxStorageOpts:
         self.exact_host_filter = exact_host_filter
         self.threads = threads
         self.all_hosts_filter: dict[str, list[str]] = all_hosts_filter or {}
+        self.cache_path = cache_path
+        self.cache_ttl = cache_ttl
+        self.recache = recache
 
     @classmethod
     def parse_params(cls, conf_params: Optional[dict[str, str]], cli_opts: Any):
@@ -41,6 +48,17 @@ class NetboxStorageOpts:
             exact_host_filter = exact_host_filter_env in ("true", "1", "t")
         else:
             exact_host_filter = bool(conf_params.get("exact_host_filter") or False)
+
+        if cache_path := os.getenv("NETBOX_CACHE_PATH"):
+            cache_path = cache_path
+        else:
+            cache_path = str(conf_params.get("cache_path", ""))
+
+        if cache_ttl := os.getenv("NETBOX_CACHE_TTL"):
+            cache_ttl = timedelta(seconds=int(cache_ttl))
+        else:
+            cache_ttl = timedelta(seconds=conf_params.get("cache_ttl", 0))
+
         return cls(
             url=url,
             token=token,
@@ -48,4 +66,7 @@ class NetboxStorageOpts:
             exact_host_filter=exact_host_filter,
             threads=int(threads),
             all_hosts_filter=all_hosts_filter,
+            cache_path=cache_path,
+            cache_ttl=cache_ttl,
+            recache=cli_opts.recache,
         )

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/provider.py

@@ -46,7 +46,8 @@ def storage_factory(opts: NetboxStorageOpts) -> Storage:
 
 class NetboxProvider(StorageProvider, AdapterWithName, AdapterWithConfig):
     def __init__(self, url: Optional[str] = None, token: Optional[str] = None, insecure: bool = False,
-                 exact_host_filter: bool = False, threads: int = 1, all_hosts_filter: dict[str, list[str]] | None = None):
+                 exact_host_filter: bool = False, threads: int = 1, all_hosts_filter: dict[str, list[str]] | None = None,
+                 cache_path: str = "", cache_ttl: int = 0):
         self.url = url
         self.token = token
         self.insecure = insecure

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/v37/storage.py

@@ -1,4 +1,7 @@
 import ssl
+from typing import Callable
+
+from requests import Session
 
 from adaptix import P
 from adaptix.conversion import get_converter, link_function, link_constant, link
@@ -23,8 +26,9 @@ class NetboxV37Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV37(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV37(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV37,
@@ -120,5 +124,6 @@ class NetboxStorageV37(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceV37, InterfaceV37, IpAddressV37, PrefixV37, FHRPGroupV37, FHRPGroupAssignmentV37]:
-        return NetboxV37Adapter(self, url, token, ssl_context, threads)
+        return NetboxV37Adapter(self, url, token, ssl_context, threads, session_factory)

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/v41/storage.py

@@ -1,4 +1,8 @@
 import ssl
+from typing import Callable
+
+from requests import Session
+
 from adaptix import P
 from adaptix.conversion import get_converter, link, link_constant, link_function
 from annetbox.v41 import client_sync
@@ -24,8 +28,9 @@ class NetboxV41Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV41(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV41(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV41,
@@ -122,5 +127,6 @@ class NetboxStorageV41(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[NetboxDeviceV41, InterfaceV41, IpAddressV41, PrefixV41, FHRPGroupV41, FHRPGroupAssignmentV41]:
-        return NetboxV41Adapter(self, url, token, ssl_context, threads)
+        return NetboxV41Adapter(self, url, token, ssl_context, threads, session_factory)

{annet-3.17.1 → annet-3.19.0}/annet/adapters/netbox/v42/storage.py

@@ -1,5 +1,7 @@
 import ssl
-from typing import Optional
+from typing import Callable, Optional
+
+from requests import Session
 
 from adaptix import P
 from adaptix.conversion import get_converter, link, link_constant, link_function
@@ -25,8 +27,9 @@ class NetboxV42Adapter(NetboxAdapter[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None,
     ):
-        self.netbox = client_sync.NetboxV42(url=url, token=token, ssl_context=ssl_context, threads=threads)
+        self.netbox = client_sync.NetboxV42(url=url, token=token, ssl_context=ssl_context, threads=threads, session_factory=session_factory)
         self.convert_device = get_converter(
             api_models.Device,
             NetboxDeviceV42,
@@ -136,11 +139,12 @@ class NetboxStorageV42(BaseNetboxStorage[
             token: str,
             ssl_context: ssl.SSLContext | None,
             threads: int,
+            session_factory: Callable[[Session], Session] | None = None,
     ) -> NetboxAdapter[
         NetboxDeviceV42, InterfaceV42, IpAddressV42, PrefixV42,
         FHRPGroupV41, FHRPGroupAssignmentV41,
     ]:
-        return NetboxV42Adapter(self, url, token, ssl_context, threads)
+        return NetboxV42Adapter(self, url, token, ssl_context, threads, session_factory)
 
     def resolve_all_vlans(self) -> list[Vlan]:
         if self._all_vlans is None:

{annet-3.17.1 → annet-3.19.0}/annet/annlib/jsontools.py

@@ -6,13 +6,16 @@ import json
 from collections.abc import Mapping, Sequence
 from itertools import starmap
 from operator import itemgetter
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, Final, Iterable, List, Optional
 
 import jsonpatch
 import jsonpointer
 from ordered_set import OrderedSet
 
 
+EVERYTHING_ACL: Final = "/*"
+
+
 def format_json(data: Any, stable: bool = False) -> str:
     """Serialize to json."""
     return json.dumps(data, indent=4, ensure_ascii=False, sort_keys=not stable) + "\n"
@@ -21,14 +24,17 @@ def format_json(data: Any, stable: bool = False) -> str:
 def apply_json_fragment(
         old: Dict[str, Any],
         new_fragment: Dict[str, Any], *,
-        acl: Sequence[str],
+        acl: Sequence[str] | None = None,
         filters: Sequence[str] | None = None,
 ) -> Dict[str, Any]:
     """
-    Replace parts of the old document with 'new_fragment' using ACL restrictions.
+    Replace parts of the old document with 'new_fragment'.
+    If `acl` is not `None`, replacement will only be made within specified keys.
     If `filter` is not `None`, only those parts which also matches at least one filter
     from the list will be modified (updated or deleted).
     """
+    if acl is None:
+        acl = [EVERYTHING_ACL]
     full_new_config = copy.deepcopy(old)
     for acl_item in acl:
         new_pointers = _resolve_json_pointers(acl_item, new_fragment)

{annet-3.17.1 → annet-3.19.0}/annet/cli_args.py

@@ -140,6 +140,12 @@ opt_tolerate_fails = Arg(
     help="Show errors without interrupting the generation"
 )
 
+opt_recache = Arg(
+    "--recache", default=False,
+    help="Force expiration of storage's local cache if it is supported"
+)
+
+
 # При параллельном запуске и включённом --tolerate-fails код возврата
 # всегда нулевой. Это не позволяет нам легко понять, прошла ли генерация
 # успешно для всех устройств. С этим флажком код будет ненулевой, если
@@ -353,6 +359,7 @@ opt_selected_context_name = Arg(
 
 # ====
 class CacheOptions(ArgGroup):
+    recache = opt_recache
     no_mesh = False
 
 

{annet-3.17.1 → annet-3.19.0}/annet/gen.py

@@ -267,11 +267,20 @@ def _old_new_per_device(ctx: OldNewDeviceContext, device: Device, filterer: Filt
             filters = filters_text.removesuffix("\n").split("\n")
 
         old_json_fragment_files = old_files.json_fragment_files.copy()
-        new_json_fragment_files = res.new_json_fragment_files(old_json_fragment_files, filters=filters)
+        new_json_fragment_files = res.new_json_fragment_files(
+            old_json_fragment_files,
+            use_acl=not ctx.args.no_acl,
+            filters=filters,
+        )
 
         if ctx.args.acl_safe:
             safe_new_files = res.new_files(safe=True)
-            safe_new_json_fragment_files = res.new_json_fragment_files(old_json_fragment_files, safe=True, filters=filters)
+            safe_new_json_fragment_files = res.new_json_fragment_files(
+                old_json_fragment_files,
+                use_acl=not ctx.args.no_acl,
+                safe=True,
+                filters=filters,
+            )
 
     if ctx.args.profile:
         perf = res.perf_mesures()

{annet-3.17.1 → annet-3.19.0}/annet/generators/result.py

@@ -85,6 +85,7 @@ class RunGeneratorResult:
     def new_json_fragment_files(
             self,
             old_files: Dict[str, Optional[str]],
+            use_acl: bool = True,
             safe: bool = False,
             filters: Sequence[str] | None = None,
     ) -> Dict[str, Tuple[Any, Optional[str]]]:
@@ -98,9 +99,12 @@ class RunGeneratorResult:
                     files[filepath] = (old_files[filepath], None)
                 else:
                     files[filepath] = ({}, None)
-            result_acl = generator_result.acl
-            if safe:
-                result_acl = generator_result.acl_safe
+            if use_acl:
+                result_acl = generator_result.acl
+                if safe:
+                    result_acl = generator_result.acl_safe
+            else:
+                result_acl = None
             previous_config: Dict[str, Any] = files[filepath][0]
             new_fragment = generator_result.config
             new_config = jsontools.apply_json_fragment(

{annet-3.17.1 → annet-3.19.0/annet.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: annet
-Version: 3.17.1
+Version: 3.19.0
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT
@@ -22,7 +22,8 @@ Requires-Dist: adaptix==3.0.0b7
 Requires-Dist: dataclass-rest==0.4
 Requires-Dist: ordered-set>=4.1.0
 Provides-Extra: netbox
-Requires-Dist: annetbox[sync]>=0.8.0; extra == "netbox"
+Requires-Dist: annetbox[sync]>=0.9.0; extra == "netbox"
+Requires-Dist: requests-cache>=1.2.1; extra == "netbox"
 Dynamic: home-page
 Dynamic: license
 Dynamic: license-file

{annet-3.17.1 → annet-3.19.0}/annet.egg-info/requires.txt

@@ -14,4 +14,5 @@ dataclass-rest==0.4
 ordered-set>=4.1.0
 
 [netbox]
-annetbox[sync]>=0.8.0
+annetbox[sync]>=0.9.0
+requests-cache>=1.2.1

{annet-3.17.1 → annet-3.19.0}/setup.py

@@ -44,7 +44,10 @@ if __name__ == "__main__":
           ],
         },
         extras_require={
-            "netbox": ["annetbox[sync]>=0.8.0"],
+            "netbox": [
+                "annetbox[sync]>=0.9.0",
+                "requests-cache>=1.2.1",
+            ],
         },
         python_requires=">=3.10",
         install_requires=requirements(),