annet 0.0__py3-none-any.whl

annet/adapters/netbox/v24/storage.py

@@ -0,0 +1,196 @@
+from logging import getLogger
+from typing import Optional, List, Union
+
+from annet.adapters.netbox.common import models
+from annet.adapters.netbox.common.manufacturer import (
+    is_supported, get_hw, get_breed,
+)
+from annet.adapters.netbox.common.query import NetboxQuery
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.storage import Storage
+from . import api_models
+from .client import NetboxV24
+
+logger = getLogger(__name__)
+
+
+def extend_device_ip(
+        ip: Optional[api_models.DeviceIp],
+) -> Optional[models.DeviceIp]:
+    if not ip:
+        return None
+    return models.DeviceIp(
+        address=ip.address,
+        id=ip.id,
+        display=ip.address,
+        family=ip.family,
+    )
+
+
+def extend_label(
+        label: Optional[api_models.Label],
+) -> Optional[models.Label]:
+    if not label:
+        return None
+    return models.Label(
+        label=label.label,
+        value=str(label.value),
+    )
+
+
+def extend_device(
+        device: api_models.Device, storage: Storage,
+) -> models.NetboxDevice:
+    manufacturer = device.device_type.manufacturer.name
+    model = device.device_type.model
+
+    return models.NetboxDevice(
+        url=device.url,
+        id=device.id,
+        name=device.name,
+        display=device.display_name,
+        device_type=device.device_type,
+        device_role=device.device_role,
+        tenant=device.tenant,
+        platform=device.platform,
+        serial=device.serial,
+        asset_tag=device.asset_tag,
+        site=device.site,
+        rack=device.rack,
+        position=device.position,
+        face=extend_label(device.face),
+        status=device.status,
+        primary_ip=extend_device_ip(device.primary_ip),
+        primary_ip4=extend_device_ip(device.primary_ip4),
+        primary_ip6=extend_device_ip(device.primary_ip6),
+        tags=[models.Entity(0, tag) for tag in device.tags],
+        custom_fields=device.custom_fields,  # ???
+        created=device.created,
+        last_updated=device.last_updated,
+
+        fqdn=device.name,
+        hostname=device.name,
+        hw=get_hw(manufacturer, model),
+        breed=get_breed(manufacturer, model),
+        interfaces=[],
+        neighbours_ids=[],
+        storage=storage,
+    )
+
+
+def extend_interface(interface: api_models.Interface) -> models.Interface:
+    return models.Interface(
+        id=interface.id,
+        name=interface.name,
+        device=interface.device,
+        enabled=interface.enabled,
+        display=interface.name,
+        ip_addresses=[],
+    )
+
+
+def extend_ip(ip: api_models.IpAddress) -> models.IpAddress:
+    return models.IpAddress(
+        id=ip.id,
+        assigned_object_id=ip.interface.id,
+        display=ip.address,
+        family=models.IpFamily(
+            value=ip.family,
+            label=str(ip.family),
+        ),
+        address=ip.address,
+        status=extend_label(ip.status),
+        tags=[models.Entity(0, tag) for tag in ip.tags],
+        created=ip.created,
+        last_updated=ip.last_updated,
+    )
+
+
+class NetboxStorageV24(Storage):
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        self.netbox = NetboxV24(
+            url=opts.url,
+            token=opts.token,
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: NetboxQuery):
+        return [
+            d.id for d in self._load_devices(query)
+        ]
+
+    def resolve_fdnds_by_query(self, query: NetboxQuery):
+        return [
+            d.name for d in self._load_devices(query)
+        ]
+
+    def make_devices(
+            self,
+            query: Union[NetboxQuery, list],
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> List[models.NetboxDevice]:
+        if isinstance(query, list):
+            query = NetboxQuery.new(query)
+        device_ids = {
+            device.id: extend_device(device=device, storage=self)
+            for device in self._load_devices(query)
+        }
+        if not device_ids:
+            return []
+
+        interfaces = self._load_interfaces(list(device_ids))
+        for interface in interfaces:
+            device_ids[interface.device.id].interfaces.append(interface)
+        return list(device_ids.values())
+
+    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
+        if not query.globs:
+            return []
+        return [
+            device
+            for device in self.netbox.all_devices().results
+            if _match_query(query, device)
+            if is_supported(device.device_type.manufacturer.name)
+        ]
+
+    def _load_interfaces(self, device_ids: List[int]) -> List[
+        models.Interface]:
+        interfaces = self.netbox.all_interfaces(device_id=device_ids)
+        extended_ifaces = {
+            interface.id: extend_interface(interface)
+            for interface in interfaces.results
+        }
+
+        ips = self.netbox.all_ip_addresses(interface_id=list(extended_ifaces))
+        for ip in ips.results:
+            extended_ip = extend_ip(ip)
+            interface = extended_ifaces[extended_ip.assigned_object_id]
+            interface.ip_addresses.append(extended_ip)
+        return list(extended_ifaces.values())
+
+    def get_device(
+            self, obj_id, preload_neighbors=False, use_mesh=None,
+            **kwargs,
+    ) -> models.NetboxDevice:
+        device = self.netbox.get_device(obj_id)
+        res = extend_device(device=device, storage=self)
+        res.interfaces = self._load_interfaces([device.id])
+        return res
+
+    def flush_perf(self):
+        pass
+
+
+def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
+    for subquery in query.globs:
+        if subquery.strip() in device_data.name:
+            return True
+    return False

annet/adapters/netbox/v37/api_models.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List, Optional, Any, Dict
+
+from annet.adapters.netbox.common.models import (
+    Entity, Label, DeviceType, DeviceIp,
+)
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+    display: str = ""  # added in 3.x
+
+
+@dataclass
+class Device(Entity):
+    url: str
+    display: str  # renamed in 3.x from display_name
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: Dict[str, Any]
+    created: datetime
+    last_updated: datetime

annet/adapters/netbox/v37/client.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+from typing import List, Optional
+
+import dateutil.parser
+from adaptix import Retort, loader
+from dataclass_rest import get
+from dataclass_rest.client_protocol import FactoryProtocol
+
+from annet.adapters.netbox.common.client import (
+    BaseNetboxClient, collect, PagingResponse,
+)
+from annet.adapters.netbox.common.models import IpAddress
+from .api_models import Device, Interface
+
+
+class NetboxV37(BaseNetboxClient):
+    def _init_response_body_factory(self) -> FactoryProtocol:
+        return Retort(recipe=[
+            loader(datetime, dateutil.parser.parse)
+        ])
+
+    @get("dcim/interfaces")
+    def interfaces(
+            self,
+            device_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Interface]:
+        pass
+
+    all_interfaces = collect(interfaces, field="device_id")
+
+    @get("ipam/ip-addresses")
+    def ip_addresses(
+            self,
+            interface_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[IpAddress]:
+        pass
+
+    all_ip_addresses = collect(ip_addresses, field="interface_id")
+
+    @get("dcim/devices")
+    def devices(
+            self,
+            name: Optional[List[str]] = None,
+            name__ic: Optional[List[str]] = None,
+            tag: Optional[List[str]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Device]:
+        pass
+
+    all_devices = collect(devices)
+
+    @get("dcim/devices/{device_id}")
+    def get_device(
+            self,
+            device_id: int,
+    ) -> Device:
+        pass

annet/adapters/netbox/v37/storage.py

@@ -0,0 +1,149 @@
+from logging import getLogger
+from typing import Optional, List, Union
+
+from adaptix import P
+from adaptix.conversion import impl_converter, link
+
+from annet.adapters.netbox.common import models
+from annet.adapters.netbox.common.manufacturer import (
+    is_supported, get_hw, get_breed,
+)
+from annet.adapters.netbox.common.query import NetboxQuery
+from annet.adapters.netbox.common.storage_opts import NetboxStorageOpts
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.storage import Storage
+from . import api_models
+from .client import NetboxV37
+
+logger = getLogger(__name__)
+
+
+@impl_converter(recipe=[
+    link(P[api_models.Device].name, P[models.NetboxDevice].hostname),
+    link(P[api_models.Device].name, P[models.NetboxDevice].fqdn),
+])
+def extend_device_base(
+        device: api_models.Device,
+        interfaces: List[models.Interface],
+        hw: Optional[HardwareView],
+        breed: str,
+        storage: Storage,
+        neighbours_ids: List[int],
+) -> models.NetboxDevice:
+    ...
+
+
+def extend_device(
+        device: api_models.Device, storage: Storage,
+) -> models.NetboxDevice:
+    return extend_device_base(
+        device=device,
+        interfaces=[],
+        breed=get_breed(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        ),
+        hw=get_hw(
+            device.device_type.manufacturer.name,
+            device.device_type.model,
+        ),
+        neighbours_ids=[],
+        storage=storage,
+    )
+
+
+@impl_converter
+def extend_interface(
+        interface: api_models.Interface, ip_addresses: List[models.IpAddress],
+) -> models.Interface:
+    ...
+
+
+class NetboxStorageV37(Storage):
+    def __init__(self, opts: Optional[NetboxStorageOpts] = None):
+        self.netbox = NetboxV37(
+            url=opts.url,
+            token=opts.token,
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        pass
+
+    def resolve_object_ids_by_query(self, query: NetboxQuery):
+        return [
+            d.id for d in self._load_devices(query)
+        ]
+
+    def resolve_fdnds_by_query(self, query: NetboxQuery):
+        return [
+            d.name for d in self._load_devices(query)
+        ]
+
+    def make_devices(
+            self,
+            query: Union[NetboxQuery, list],
+            preload_neighbors=False,
+            use_mesh=None,
+            preload_extra_fields=False,
+            **kwargs,
+    ) -> List[models.NetboxDevice]:
+        if isinstance(query, list):
+            query = NetboxQuery.new(query)
+        device_ids = {
+            device.id: extend_device(device=device, storage=self)
+            for device in self._load_devices(query)
+        }
+        if not device_ids:
+            return []
+
+        interfaces = self._load_interfaces(list(device_ids))
+        for interface in interfaces:
+            device_ids[interface.device.id].interfaces.append(interface)
+        return list(device_ids.values())
+
+    def _load_devices(self, query: NetboxQuery) -> List[api_models.Device]:
+        if not query.globs:
+            return []
+        return [
+            device
+            for device in self.netbox.all_devices(
+                name__ic=query.globs,
+            ).results
+            if _match_query(query, device)
+            if is_supported(device.device_type.manufacturer.name)
+        ]
+
+    def _load_interfaces(self, device_ids: List[int]) -> List[
+        models.Interface]:
+        interfaces = self.netbox.all_interfaces(device_id=device_ids)
+        extended_ifaces = {
+            interface.id: extend_interface(interface, [])
+            for interface in interfaces.results
+        }
+
+        ips = self.netbox.all_ip_addresses(interface_id=list(extended_ifaces))
+        for ip in ips.results:
+            extended_ifaces[ip.assigned_object_id].ip_addresses.append(ip)
+        return list(extended_ifaces.values())
+
+    def get_device(
+            self, obj_id, preload_neighbors=False, use_mesh=None,
+            **kwargs,
+    ) -> models.NetboxDevice:
+        device = self.netbox.get_device(obj_id)
+        res = extend_device(device=device, storage=self)
+        res.interfaces = self._load_interfaces([device.id])
+        return res
+
+    def flush_perf(self):
+        pass
+
+
+def _match_query(query: NetboxQuery, device_data: api_models.Device) -> bool:
+    for subquery in query.globs:
+        if subquery.strip() in device_data.name:
+            return True
+    return False

annet/annet.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import sys
+
+import annet
+from annet import argparse, cli, generators, hardware, lib, rulebook
+
+
+# =====
+@lib.catch_ctrl_c
+def main():
+    annet.assert_python_version()
+    parser = argparse.ArgParser()
+    cli.fill_base_args(parser, annet.__name__, "configs/logging.yaml")
+    rulebook.rulebook_provider_connector.set(rulebook.DefaultRulebookProvider)
+    hardware.hardware_connector.set(hardware.AnnetHardwareProvider)
+
+    parser.add_commands(parser.find_subcommands(cli.list_subcommands()))
+    try:
+        return parser.dispatch(pre_call=annet.init, add_help_command=True)
+    except (generators.GeneratorError, annet.ExecError):
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

annet/annlib/__init__.py

@@ -0,0 +1,7 @@
+import os
+
+import colorama
+
+# отключить colorama.init, если стоит env-переменная. Нужно в тестах
+if os.environ.get("ANN_FORCE_COLOR", None) not in [None, "", "0", "no"]:
+    colorama.init = lambda *_, **__: None

annet/annlib/command.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass, field
+
+from typing import List, Optional
+
+
+FIRST_EXCEPTION = 1
+ALL_COMPLETED = 2
+
+
+@dataclass
+class Question:
+    question: str  # frame it using / if it is a regular expression
+    answer: str
+    is_regexp: Optional[bool] = False
+
+
+@dataclass
+class Command:
+    cmd: str
+    questions: Optional[List[Question]] = None
+    exc_handler: Optional[List[Question]] = None
+    timeout: Optional[int] = None
+    suppress_nonzero: bool = False
+    suppress_eof: bool = False
+
+    def __str__(self) -> str:
+        return self.cmd
+
+
+@dataclass
+class CommandList:
+    cmss: List[Command] = field(default_factory=list)
+
+    def __post_init__(self):
+        if not self.cmss:
+            self.cmss = []
+
+    def __iter__(self):
+        return iter(self.cmss)
+
+    def __len__(self) -> int:
+        return len(self.cmss)
+
+    def add_cmd(self, cmd: Command) -> None:
+        assert isinstance(cmd, Command)
+        self.cmss.append(cmd)
+
+    def as_list(self) -> List[Command]:  # TODO: delete
+        return self.cmss

annet/annlib/diff.py

@@ -0,0 +1,158 @@
+import functools
+import ipaddress
+from typing import Dict, Generator
+
+import colorama
+
+from .types import Diff, Op
+
+# NOCDEV-1720
+
+
+diff_ops = {
+    "+": Op.ADDED,
+    "-": Op.REMOVED,
+    " ": Op.AFFECTED,
+    ">": Op.MOVED,
+}
+
+ops_sign = {
+    v: k
+    for k, v in diff_ops.items()
+}
+
+ops_order = {
+    Op.AFFECTED: 0,
+    Op.MOVED: 1,
+    Op.REMOVED: 2,
+    Op.ADDED: 3,
+}
+
+ops_color = {
+    Op.REMOVED: colorama.Fore.RED,
+    Op.ADDED: colorama.Fore.GREEN,
+    Op.AFFECTED: colorama.Fore.CYAN,
+    Op.MOVED: colorama.Fore.YELLOW,
+}
+
+
+def is_int(ts):
+    try:
+        int(ts)
+        return True
+    except ValueError:
+        return False
+
+
+def is_ip(ts):
+    try:
+        ipaddress.ip_interface(ts)
+        return True
+    except ValueError:
+        return False
+
+
+def diff_cmp(diff_l, diff_r):
+    """
+Сборник костылей для сравнения двух строк диффа
+    """
+    (op_l, line_l, _, _) = diff_l
+    (op_r, line_r, _, _) = diff_r
+
+    cmp_line = (line_l > line_r) - (line_l < line_r)
+    cmp_op = ops_order[op_l] - ops_order[op_r]
+    if cmp_line == 0:
+        # При равенстве строк порядок определяется операцией
+        return cmp_op
+
+    if cmp_op == 0:
+        # Если для строк операции одинаковы, то считаем их равными
+        # По идее TimSort стабилен и порядок просто не должен измениться
+        return 0
+
+    # Для частично совпадающих строк
+    lws = line_l.split(" ")
+    rws = line_r.split(" ")
+    res = 0
+    for i, lw in enumerate(lws):
+        if len(rws) > i:
+            rw = rws[i]
+            if is_int(lw) and is_int(rw):
+                # В одинаковом положении в строках есть инты, сортируем по ним
+                res = int(lw) - int(rw)
+                if res == 0:
+                    # При равных интах - сортируем по операции
+                    res = cmp_op
+            elif is_ip(lw) and is_ip(rw):
+                # Аналогично интам обрабатываем ip-адреса
+                ip_l = ipaddress.ip_interface(lw)
+                ip_r = ipaddress.ip_interface(rw)
+                try:
+                    res = (ip_l > ip_r) - (ip_l < ip_r)
+                except TypeError:
+                    res = 1
+                    if ip_l.version == 4:
+                        res = -1
+                if res == 0:
+                    res = cmp_op
+            elif i > 0:
+                if lw == rw:
+                    res = cmp_op
+            else:
+                continue
+            break
+    if res != 0:
+        return res
+    return cmp_line
+
+
+def resort_diff(diff: Diff) -> Diff:
+    res = []
+    df = sorted(diff, key=functools.cmp_to_key(diff_cmp))
+    for line in df:
+        ln = line
+        if len(line[2]) > 0:
+            ln = (line[0], line[1], resort_diff(line[2]), line[3])
+        res.append(ln)
+    return res
+
+
+def colorize_line_with_color(line: str, color: int, no_color: bool):
+    stripped = line.rstrip("\n")
+    add_newlines = len(line) - len(stripped)
+    line = stripped
+
+    if not no_color:
+        line = "%s%s%s%s" % (colorama.Style.BRIGHT, color, line, colorama.Style.RESET_ALL)
+
+    line += "\n" * add_newlines
+    return line
+
+
+def colorize_line(line, no_color=False):
+    op = diff_ops[line[0]]
+    color = ops_color[op]
+    return colorize_line_with_color(line, color, no_color)
+
+
+def gen_pre_as_diff(
+    pre: Dict,
+    show_rules: bool,
+    indent: str,
+    no_color: bool,
+    _level: int = 0
+) -> Generator[str, None, None]:
+    ops = [(order, op) for op, order in ops_order.items()]
+    ops.sort()
+    for (raw_rule, content) in pre.items():
+        items = content["items"].items()
+        for (_, diff) in items:  # pylint: disable=redefined-outer-name
+            if show_rules and not raw_rule == "__MULTILINE_BODY__":
+                line = "# %s%s\n" % (indent * _level, raw_rule)
+                yield colorize_line_with_color(line, colorama.Fore.BLACK, no_color)
+            for (op, rows) in [(op, diff[op]) for (_, op) in ops]:
+                for item in rows:
+                    line = "%s%s %s\n" % (ops_sign[op], indent * _level, item["row"])
+                    yield colorize_line_with_color(line, ops_color[op], no_color)
+                    if len(item["children"]) != 0:
+                        yield from gen_pre_as_diff(item["children"], show_rules, indent, no_color, _level + 1)

annet/annlib/errors.py

@@ -0,0 +1,8 @@
+class ExecError(Exception):
+    """Обработчик этого exception должен залоггировать ошибку и выйти с exit_code 1"""
+    pass
+
+
+class DeployCancelled(Exception):
+    """Деплой на устройство был отменен"""
+    pass