annet 0.6__py3-none-any.whl → 0.8__py3-none-any.whl

annet/adapters/netbox/v37/api_models.py

@@ -0,0 +1,37 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List, Optional, Any
+
+from annet.adapters.netbox.common.models import (
+    Entity, Label, DeviceType, DeviceIp,
+)
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+    display: str = ""  # added in 3.x
+
+
+@dataclass
+class Device(Entity):
+    display: str  # renamed in 3.x from display_name
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: dict[str, Any]
+    created: datetime
+    last_updated: datetime

annet/adapters/netbox/v37/client.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+from typing import List, Optional
+
+import dateutil.parser
+from adaptix import Retort, loader
+from dataclass_rest import get
+from dataclass_rest.client_protocol import FactoryProtocol
+
+from annet.adapters.netbox.common.client import (
+    BaseNetboxClient, collect, PagingResponse,
+)
+from annet.adapters.netbox.common.models import IpAddress
+from .api_models import Device, Interface
+
+
+class NetboxV37(BaseNetboxClient):
+    def _init_response_body_factory(self) -> FactoryProtocol:
+        return Retort(recipe=[
+            loader(datetime, dateutil.parser.parse)
+        ])
+
+    @get("dcim/interfaces")
+    def interfaces(
+            self,
+            device_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Interface]:
+        pass
+
+    all_interfaces = collect(interfaces, field="device_id")
+
+    @get("ipam/ip-addresses")
+    def ip_addresses(
+            self,
+            interface_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[IpAddress]:
+        pass
+
+    all_ip_addresses = collect(ip_addresses, field="interface_id")
+
+    @get("dcim/devices")
+    def devices(
+            self,
+            name: Optional[List[str]] = None,
+            name__ic: Optional[List[str]] = None,
+            tag: Optional[List[str]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Device]:
+        pass
+
+    all_devices = collect(devices)
+
+    @get("dcim/devices/{device_id}")
+    def get_device(
+            self,
+            device_id: int,
+    ) -> Device:
+        pass

annet/adapters/netbox/v37/storage.py

@@ -0,0 +1,143 @@
+from logging import getLogger
+from typing import Optional, List
+
+from adaptix import P
+from adaptix.conversion import impl_converter, link
+
+from annet.adapters.netbox.common import models
+from annet.adapters.netbox.common.manufacturer import (
+    is_supported, get_hw, get_breed,
+)
+from annet.adapters.netbox.common.query import NetboxQuery
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.storage import Storage
+from . import api_models
+from .client import NetboxV37
+
+logger = getLogger(__name__)
+
+
+@impl_converter(recipe=[
+    link(P[api_models.Device].name, P[models.NetboxDevice].hostname),
+    link(P[api_models.Device].name, P[models.NetboxDevice].fqdn),
+])
+def extend_device_base(
+        device: api_models.Device,
+        interfaces: List[models.Interface],
+        hw: Optional[HardwareView],
+        breed: str,
+        neighbours_ids: List[int],
+) -> models.NetboxDevice:
+    ...
+
+
+def extend_device(
+        device: api_models.Device,
+) -> models.NetboxDevice:
+    return extend_device_base(
+        device=device,
+        interfaces=[],
+        breed=get_breed(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        ),
+        hw=get_hw(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        ),
+        neighbours_ids=[],
+    )
+
+
+@impl_converter
+def extend_interface(
+        interface: api_models.Interface, ip_addresses: List[models.IpAddress],
+) -> models.Interface:
+    ...
+
+
+class NetboxStorageV37(Storage):
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        self.netbox = NetboxV37(
+            url=opts.url,
+            token=opts.token,
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: NetboxQuery):
+        return [
+            d.id for d in self._load_devices(query)
+        ]
+
+    def resolve_fdnds_by_query(self, query: NetboxQuery):
+        return [
+            d.name for d in self._load_devices(query)
+        ]
+
+    def make_devices(
+            self,
+            query: NetboxQuery,
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> List[models.NetboxDevice]:
+        device_ids = {
+            device.id: extend_device(device=device)
+            for device in self._load_devices(query)
+        }
+        if not device_ids:
+            return []
+
+        interfaces = self._load_interfaces(list(device_ids))
+        for interface in interfaces:
+            device_ids[interface.device.id].interfaces.append(interface)
+        return list(device_ids.values())
+
+    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
+        return [
+            device
+            for device in self.netbox.all_devices(
+                name__ic=query.globs,
+            ).results
+            if _match_query(query, device)
+            if is_supported(device.device_type.manufacturer.name)
+        ]
+
+    def _load_interfaces(self, device_ids: List[int]) -> List[
+        models.Interface]:
+        interfaces = self.netbox.all_interfaces(device_id=device_ids)
+        extended_ifaces = {
+            interface.id: extend_interface(interface, [])
+            for interface in interfaces.results
+        }
+
+        ips = self.netbox.all_ip_addresses(interface_id=list(extended_ifaces))
+        for ip in ips.results:
+            extended_ifaces[ip.assigned_object_id].ip_addresses.append(ip)
+        return list(extended_ifaces.values())
+
+    def get_device(
+            self, obj_id, preload_neighbors=False, use_mesh=None,
+            **kwargs,
+    ) -> models.NetboxDevice:
+        device = self.netbox.get_device(obj_id)
+        res = extend_device(device=device)
+        res.interfaces = self._load_interfaces([device.id])
+        return res
+
+    def flush_perf(self):
+        pass
+
+
+def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
+    for subquery in query.globs:
+        if subquery.strip() in device_data.name:
+            return True
+    return False

annet/annlib/jsontools.py

@@ -87,3 +87,26 @@ def apply_patch(content: Optional[bytes], patch_bytes: bytes) -> bytes:
 
     new_contents = format_json(new_doc, stable=True).encode()
     return new_contents
+
+
+def apply_acl_filters(content: Dict[str, Any], filters: List[str]) -> Dict[str, Any]:
+    result = {}
+    for f in filters:
+        pointer = jsonpointer.JsonPointer(f)
+
+        try:
+            part = pointer.get(copy.deepcopy(content))
+
+            sub_tree = result
+            for i in pointer.get_parts():
+                if i not in sub_tree:
+                    sub_tree[i] = {}
+                sub_tree = sub_tree[i]
+
+            patch = jsonpatch.JsonPatch([{"op": "add", "path": f, "value": part}])
+            result = patch.apply(result)
+        except jsonpointer.JsonPointerException:
+            # no value found in new_fragment by the pointer, skip the ACL item
+            continue
+
+    return result

annet/api/__init__.py

@@ -188,7 +188,9 @@ def _print_pre_as_diff(pre, show_rules, indent, file=None, _level=0):
 def log_host_progress_cb(pool: Parallel, task_result: TaskResult):
     progress_logger = get_logger("progress")
     args = cast(cli_args.QueryOptions, pool.args[0])
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         hosts = storage.resolve_fdnds_by_query(args.query)
     perc = int(pool.tasks_done / len(hosts) * 100)
     fqdn = hosts[task_result.device_id]
@@ -210,7 +212,9 @@ def log_host_progress_cb(pool: Parallel, task_result: TaskResult):
 # =====
 def gen(args: cli_args.ShowGenOptions):
     """ Сгенерировать конфиг для устройств """
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         loader = ann_gen.Loader(storage, args)
         stdin = args.stdin(storage=storage, filter_acl=args.filter_acl, config=None)
 
@@ -243,7 +247,9 @@ def _diff_files(old_files, new_files, context=3):
 def patch(args: cli_args.ShowPatchOptions):
     """ Сгенерировать патч для устройств """
     global live_configs  # pylint: disable=global-statement
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         loader = ann_gen.Loader(storage, args)
         if args.config == "running":
             fetcher = annet.deploy.fetcher_connector.get()
@@ -286,7 +292,9 @@ def _patch_worker(device_id, args: cli_args.ShowPatchOptions, stdin, loader: ann
 # =====
 def res_diff_patch(device_id, args: cli_args.ShowPatchOptions, stdin, loader: ann_gen.Loader, filterer: filtering.Filterer) -> Iterable[
     Tuple[OldNewResult, Dict, Dict]]:
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         for res in ann_gen.old_new(
             args,
             storage,
@@ -314,7 +322,9 @@ def res_diff_patch(device_id, args: cli_args.ShowPatchOptions, stdin, loader: an
 
 def diff(args: cli_args.DiffOptions, loader: ann_gen.Loader, filterer: filtering.Filterer) -> Mapping[Device, Union[Diff, PCDiff]]:
     ret = {}
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         for res in ann_gen.old_new(
             args,
             storage,
@@ -624,7 +634,9 @@ def deploy(args: cli_args.DeployOptions) -> ExitCode:
     """ Сгенерировать конфиг для устройств и задеплоить его """
     ret: ExitCode = 0
     deployer = Deployer(args)
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         global live_configs  # pylint: disable=global-statement
         loader = ann_gen.Loader(storage, args)
         filterer = filtering.filterer_connector.get()

annet/cli.py

@@ -54,7 +54,9 @@ def show_current(args: cli_args.QueryOptions, config, arg_out: cli_args.FileOutO
                         entire_data = ""
                     yield (output_driver.entire_config_dest_path(device, entire_path), entire_data, False)
 
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         ids = storage.resolve_object_ids_by_query(args.query)
         if not ids:
             get_logger().error("No devices found for %s", args.query)
@@ -80,7 +82,9 @@ def gen(args: cli_args.ShowGenOptions):
 @subcommand(cli_args.ShowDiffOptions)
 def diff(args: cli_args.ShowDiffOptions):
     """ Сгенерировать конфиг для устройств и показать дифф по рулбуку с текущим """
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         filterer = filtering.filterer_connector.get()
         loader = Loader(storage, args)
         output_driver_connector.get().write_output(

annet/cli_args.py

@@ -433,6 +433,11 @@ class FileOutOptions(ArgGroup):
     no_label = opt_no_label
     no_color = opt_no_color
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.dest:
+            self.no_color = True
+
 
 class DiffOptions(GenOptions, ComocutorOptions):
     clear = opt_clear
@@ -467,6 +472,11 @@ class ShowDiffOptions(DiffOptions, FileOutOptions):
     show_rules = opt_show_rules
     no_collapse = opt_no_collapse
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.dest:
+            self.no_collapse = True
+
 
 class ShowPatchOptions(PatchOptions, FileOutOptions):
     indent = opt_indent

annet/diff.py

@@ -30,8 +30,7 @@ def gen_sort_diff(
     :param diffs: Маппинг устройства в дифф
     :param args: Параметры коммандной строки
     """
-    # NOCDEV-2201 non-null --dest implies --no-collapse
-    if args.no_collapse or args.dest:
+    if args.no_collapse:
         devices_to_diff = {(dev,): diff for dev, diff in diffs.items()}
     else:
         non_pc_diffs = {dev: diff for dev, diff in diffs.items() if not isinstance(diff, PCDiff)}

annet/gen.py

@@ -182,6 +182,7 @@ def _old_new_per_device(ctx: OldNewDeviceContext, device: Device, filterer: Filt
         if not ctx.args.no_acl:
             acl_rules = generators.compile_acl_text(res.acl_text(), device.hw.vendor)
             old = (old and patching.apply_acl(old, acl_rules))
+
             new = patching.apply_acl(
                 new,
                 acl_rules,
@@ -232,11 +233,29 @@ def _old_new_per_device(ctx: OldNewDeviceContext, device: Device, filterer: Filt
 
         entire_results = res.entire_results
         json_fragment_results = res.json_fragment_results
+        old_json_fragment_files = old_files.json_fragment_files
+
         new_files = res.new_files()
-        new_json_fragment_files = res.new_json_fragment_files(old_files.json_fragment_files)
+        new_json_fragment_files = res.new_json_fragment_files(old_json_fragment_files)
+
         if ctx.args.acl_safe:
             safe_new_files = res.new_files(safe=True)
 
+        filters = build_filter_text(filterer, device, ctx.stdin, ctx.args, ctx.config).split("\n")
+
+        for file_name in new_json_fragment_files:
+            new_json_fragment_files = _update_json_config(
+                new_json_fragment_files,
+                file_name,
+                jsontools.apply_acl_filters(new_json_fragment_files[file_name][0], filters)
+            )
+        for file_name in old_json_fragment_files:
+            old_json_fragment_files = _update_json_config(
+                old_json_fragment_files,
+                file_name,
+                jsontools.apply_acl_filters(old_json_fragment_files[file_name][0], filters)
+            )
+
     if ctx.args.profile:
         perf = res.perf_mesures()
         combined_perf[ALL_GENS] = {"total": time.monotonic() - start}
@@ -253,7 +272,7 @@ def _old_new_per_device(ctx: OldNewDeviceContext, device: Device, filterer: Filt
         new_files=new_files,
         partial_result=partial_results,
         entire_result=entire_results,
-        old_json_fragment_files=old_files.json_fragment_files,
+        old_json_fragment_files=old_json_fragment_files,
         new_json_fragment_files=new_json_fragment_files,
         json_fragment_result=json_fragment_results,
         implicit_rules=implicit_rules,
@@ -266,6 +285,13 @@ def _old_new_per_device(ctx: OldNewDeviceContext, device: Device, filterer: Filt
     )
 
 
+def _update_json_config(json_files, file_name, new_config):
+    file = list(json_files[file_name])
+    file[0] = new_config
+    json_files[file_name] = tuple(file)
+    return json_files
+
+
 @dataclasses.dataclass
 class DeviceDownloadedFiles:
     # map file path to file content for entire generators
@@ -425,7 +451,9 @@ def worker(device_id, args: ShowGenOptions, stdin, loader: "Loader", filterer: F
     if span:
         span.set_attribute("device.id", device_id)
 
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         for res in old_new(
             args,
             storage,
@@ -465,7 +493,9 @@ def worker(device_id, args: ShowGenOptions, stdin, loader: "Loader", filterer: F
 
 
 def old_new_worker(device_id, args: DeployOptions, config, stdin, loader: "Loader", filterer: Filterer):
-    with storage_connector.get().storage()(args) as storage:
+    connector = storage_connector.get()
+    storage_opts = connector.opts().from_cli_opts(args)
+    with connector.storage()(storage_opts) as storage:
         yield from old_new(
             args,
             storage,