annet 0.0__py3-none-any.whl

annet/__init__.py

@@ -0,0 +1,61 @@
+import logging
+import logging.config
+import os
+import pkgutil
+import sys
+from argparse import SUPPRESS, Namespace
+
+import colorama
+import yaml
+from annet.annlib.errors import (  # pylint: disable=wrong-import-position
+    DeployCancelled,
+    ExecError,
+)
+from contextlog import patch_logging, patch_threading
+from valkit.python import valid_logging_level
+
+import annet.argparse
+
+
+__all__ = ("DeployCancelled", "ExecError")
+
+DEBUG2_LEVELV_NUM = 9
+
+
+def fill_base_args(parser: annet.argparse.ArgParser, pkg_name: str, logging_config: str):
+    parser.add_argument("--log-level", default="WARN", type=valid_logging_level,
+                        help="Уровень детализации логов (DEBUG, DEBUG2 (with comocutor debug), INFO, WARN, CRITICAL)")
+    parser.add_argument("--pkg_name", default=pkg_name, help=SUPPRESS)
+    parser.add_argument("--logging_config", default=logging_config, help=SUPPRESS)
+
+
+def init_logging(options: Namespace):
+    patch_logging()
+    patch_threading()
+    logging.captureWarnings(True)
+    logging_config = yaml.safe_load(pkgutil.get_data(options.pkg_name, options.logging_config))
+    if options.log_level is not None:
+        logging_config.setdefault("root", {})
+        logging_config["root"]["level"] = options.log_level
+    logging.addLevelName(DEBUG2_LEVELV_NUM, "DEBUG2")
+    logging.config.dictConfig(logging_config)
+
+
+def init(options: Namespace):
+    init_logging(options)
+
+    # Отключить colorama.init, если стоит env-переменная. Нужно в тестах
+    if os.environ.get("ANN_FORCE_COLOR", None) not in [None, "", "0", "no"]:
+        colorama.init = lambda *_, **__: None
+    colorama.init()
+
+    # Workaround for Python 3.8.0: https://bugs.python.org/issue38529
+    import asyncio.streams
+    if hasattr(asyncio.streams.StreamReaderProtocol, "_on_reader_gc"):
+        asyncio.streams.StreamReaderProtocol._on_reader_gc = lambda *args, **kwargs: None  # pylint: disable=protected-access
+
+
+def assert_python_version():
+    if sys.version_info < (3, 8, 0):
+        sys.stderr.write("Error: you need python 3.8.0 or higher\n")
+        sys.exit(1)

annet/adapters/netbox/common/client.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass
+from functools import wraps
+from typing import Generic, Optional, List, TypeVar, Callable
+
+from dataclass_rest.http.requests import RequestsClient
+from requests import Session
+
+Model = TypeVar("Model")
+
+
+@dataclass
+class PagingResponse(Generic[Model]):
+    next: Optional[str]
+    previous: Optional[str]
+    count: int
+    results: List[Model]
+
+
+Func = TypeVar("Func", bound=Callable)
+
+
+def _collect_by_pages(func: Func) -> Func:
+    """Collect all results using only pagination."""
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        kwargs.setdefault("offset", 0)
+        limit = kwargs.setdefault("limit", 100)
+        results = []
+        method = func.__get__(self, self.__class__)
+        has_next = True
+        while has_next:
+            page = method(*args, **kwargs)
+            kwargs["offset"] += limit
+            results.extend(page.results)
+            has_next = bool(page.next)
+        return PagingResponse(
+            previous=None,
+            next=None,
+            count=len(results),
+            results=results,
+        )
+
+    return wrapper
+
+
+# default batch size 100 is calculated to fit list of UUIDs in 4k URL length
+def collect(func: Func, field: str = "", batch_size: int = 100) -> Func:
+    """
+    Collect data from method iterating over pages and filter batches.
+
+    :param func: Method to call
+    :param field: Field which defines a filter split into batches
+    :param batch_size: Limit of values in `field` filter requested at a time
+    """
+    func = _collect_by_pages(func)
+    if not field:
+        return func
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        value = kwargs.get(field)
+        if not value:
+            return func(*args, **kwargs)
+
+        method = func.__get__(self, self.__class__)
+        results = []
+        for offset in range(0, len(value), batch_size):
+            kwargs[field] = value[offset:offset + batch_size]
+            page = method(*args, **kwargs)
+            results.extend(page.results)
+        return PagingResponse(
+            previous=None,
+            next=None,
+            count=len(results),
+            results=results,
+        )
+
+    return wrapper
+
+
+class BaseNetboxClient(RequestsClient):
+    def __init__(self, url: str, token: str):
+        url = url.rstrip("/") + "/api/"
+        session = Session()
+        if token:
+            session.headers["Authorization"] = f"Token {token}"
+        super().__init__(url, session)

annet/adapters/netbox/common/manufacturer.py

@@ -0,0 +1,62 @@
+from logging import getLogger
+
+from annet.annlib.netdev.views.hardware import HardwareView
+
+logger = getLogger(__name__)
+
+_VENDORS = {
+    "cisco": "Cisco",
+    "catalyst": "Cisco Catalyst",
+    "nexus": "Cisco Nexus",
+    "huawei": "Huawei",
+    "juniper": "Juniper",
+    "arista": "Arista",
+    "pc": "PC",
+    "nokia": "Nokia",
+    "aruba": "Aruba",
+    "routeros": "RouterOS",
+    "ribbon": "Ribbon",
+}
+
+
+def _vendor_to_hw(vendor):
+    return HardwareView(_VENDORS.get(vendor.lower(), vendor), None)
+
+
+def get_hw(manufacturer: str, model: str):
+    # by some reason Netbox calls Mellanox SN as MSN, so we fix them here
+    if manufacturer == "Mellanox" and model.startswith("MSN"):
+        model = model.replace("MSN", "SN", 1)
+    hw = _vendor_to_hw(manufacturer + " " + model)
+    if not hw:
+        raise ValueError(f"unsupported manufacturer {manufacturer}")
+    return hw
+
+
+def get_breed(manufacturer: str, model: str):
+    if manufacturer == "Huawei" and model.startswith("CE"):
+        return "vrp85"
+    elif manufacturer == "Huawei" and model.startswith("NE"):
+        return "vrp85"
+    elif manufacturer == "Huawei":
+        return "vrp55"
+    elif manufacturer == "Mellanox":
+        return "cuml2"
+    elif manufacturer == "Juniper":
+        return "jun10"
+    elif manufacturer == "Cisco":
+        return "ios12"
+    elif manufacturer == "Adva":
+        return "adva8"
+    elif manufacturer == "Arista":
+        return "eos4"
+    raise ValueError(f"unsupported manufacturer {manufacturer}")
+
+
+def is_supported(manufacturer: str) -> bool:
+    if manufacturer not in (
+            "Huawei", "Mellanox", "Juniper", "Cisco", "Adva", "Arista",
+    ):
+        logger.warning("Unsupported manufacturer `%s`", manufacturer)
+        return False
+    return True

annet/adapters/netbox/common/models.py

@@ -0,0 +1,105 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional, Any, Dict
+
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.storage import Storage
+
+
+@dataclass
+class Entity:
+    id: int
+    name: str
+
+
+@dataclass
+class Label:
+    value: str
+    label: str
+
+
+@dataclass
+class IpFamily:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceType:
+    id: int
+    manufacturer: Entity
+    model: str
+
+
+@dataclass
+class DeviceIp:
+    id: int
+    display: str
+    address: str
+    family: int
+
+
+@dataclass
+class IpAddress:
+    id: int
+    assigned_object_id: int
+    display: str
+    family: IpFamily
+    address: str
+    status: Label
+    tags: List[Entity]
+    created: datetime
+    last_updated: datetime
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+    display: str = ""
+    ip_addresses: List[IpAddress] = field(default_factory=list)
+
+
+@dataclass
+class NetboxDevice(Entity):
+    url: str
+    storage: Storage
+    neighbours_ids: List[int]
+
+    display: str
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[Entity]
+    custom_fields: Dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+    fqdn: str
+    hostname: str
+    hw: Optional[HardwareView]
+    breed: str
+
+    interfaces: List[Interface]
+
+    # compat
+
+    def __hash__(self):
+        return hash((self.id, type(self)))
+
+    def __eq__(self, other):
+        return type(self) is type(other) and self.url == other.url
+
+    def is_pc(self):
+        return self.device_type.manufacturer.name == "Mellanox"

annet/adapters/netbox/common/query.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import List, Union, Iterable, Optional
+
+from annet.storage import Query
+
+
+@dataclass
+class NetboxQuery(Query):
+    query: List[str]
+
+    @classmethod
+    def new(
+            cls, query: Union[str, Iterable[str]],
+            hosts_range: Optional[slice] = None,
+    ) -> "NetboxQuery":
+        if hosts_range is not None:
+            raise ValueError("host_range is not supported")
+        return cls(query=list(query))
+
+    @property
+    def globs(self):
+        # We process every query host as a glob
+        return self.query

annet/adapters/netbox/common/status_client.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from typing import Dict
+
+from adaptix import Retort, name_mapping, NameStyle
+from dataclass_rest import get
+from dataclass_rest.client_protocol import FactoryProtocol
+
+from .client import BaseNetboxClient
+
+
+@dataclass
+class Status:
+    netbox_version: str
+    plugins: Dict[str, str]
+
+
+class NetboxStatusClient(BaseNetboxClient):
+    def _init_response_body_factory(self) -> FactoryProtocol:
+        return Retort(recipe=[
+            name_mapping(name_style=NameStyle.LOWER_KEBAB)
+        ])
+
+    @get("status")
+    def status(self) -> Status:
+        ...

annet/adapters/netbox/common/storage_opts.py

@@ -0,0 +1,14 @@
+import os
+
+
+class NetboxStorageOpts:
+    def __init__(self, url: str, token: str):
+        self.url = url
+        self.token = token
+
+    @classmethod
+    def from_cli_opts(cls, cli_opts):
+        return cls(
+            url=os.getenv("NETBOX_URL", "http://localhost"),
+            token=os.getenv("NETBOX_TOKEN", "").strip(),
+        )

annet/adapters/netbox/provider.py

@@ -0,0 +1,34 @@
+from dataclass_rest.exceptions import ClientError
+
+from annet.storage import StorageProvider, Storage
+from .common.status_client import NetboxStatusClient
+from .common.storage_opts import NetboxStorageOpts
+from .common.query import NetboxQuery
+from .v24.storage import NetboxStorageV24
+from .v37.storage import NetboxStorageV37
+
+
+def storage_factory(opts: NetboxStorageOpts) -> Storage:
+    client = NetboxStatusClient(opts.url, opts.token)
+    try:
+        status = client.status()
+    except ClientError as e:
+        if e.status_code == 404:
+            # old version do not support status reqeust
+            return NetboxStorageV24(opts)
+        raise
+    if status.netbox_version.startswith("3."):
+        return NetboxStorageV37(opts)
+    else:
+        raise ValueError(f"Unsupported version: {status.netbox_version}")
+
+
+class NetboxProvider(StorageProvider):
+    def storage(self):
+        return storage_factory
+
+    def opts(self):
+        return NetboxStorageOpts
+
+    def query(self):
+        return NetboxQuery

annet/adapters/netbox/v24/api_models.py

@@ -0,0 +1,73 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List, Optional, Any, Dict
+
+from annet.adapters.netbox.common.models import Entity, DeviceType
+
+
+@dataclass
+class Label:
+    value: int
+    label: str
+
+
+@dataclass
+class DeviceIp:
+    id: int
+    address: str
+    family: int
+
+
+@dataclass
+class Device(Entity):
+    url: str
+    display_name: str
+    device_type: DeviceType
+    device_role: Entity
+    tenant: Optional[Entity]
+    platform: Optional[Entity]
+    serial: str
+    asset_tag: Optional[str]
+    site: Entity
+    rack: Optional[Entity]
+    position: Optional[float]
+    face: Optional[Label]
+    status: Label
+    primary_ip: Optional[DeviceIp]
+    primary_ip4: Optional[DeviceIp]
+    primary_ip6: Optional[DeviceIp]
+    tags: List[str]
+    custom_fields: Dict[str, Any]
+    created: datetime
+    last_updated: datetime
+
+
+@dataclass
+class Interface(Entity):
+    device: Entity
+    enabled: bool
+
+
+@dataclass
+class Vrf(Entity):
+    rd: str
+
+
+@dataclass
+class IpAddress:
+    id: int
+    family: int
+    address: str
+    vrf: Optional[Vrf]
+    tenant: Any  # ???
+    status: Label
+    description: Optional[str]
+    custom_fields: Dict[str, Any]
+    tags: List[str]
+    created: datetime
+    last_updated: datetime
+
+    interface: Entity
+
+    nat_inside: Any  # ???
+    nat_outside: Any  # ???

annet/adapters/netbox/v24/client.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+from typing import List, Optional
+
+import dateutil.parser
+from adaptix import Retort, loader
+from dataclass_rest import get
+
+from annet.adapters.netbox.common.client import (
+    BaseNetboxClient, collect, PagingResponse,
+)
+from .api_models import Device, Interface, IpAddress
+
+
+class NetboxV24(BaseNetboxClient):
+    def _init_response_body_factory(self) -> Retort:
+        return Retort(recipe=[
+            loader(datetime, dateutil.parser.parse)
+        ])
+
+    @get("dcim/interfaces")
+    def interfaces(
+            self,
+            device_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Interface]:
+        pass
+
+    all_interfaces = collect(interfaces, field="device_id")
+
+    @get("ipam/ip-addresses")
+    def ip_addresses(
+            self,
+            interface_id: Optional[List[int]] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[IpAddress]:
+        pass
+
+    all_ip_addresses = collect(ip_addresses, field="interface_id")
+
+    @get("dcim/devices")
+    def devices(
+            self,
+            name: Optional[List[str]] = None,
+            tag: Optional[str] = None,
+            limit: int = 20,
+            offset: int = 0,
+    ) -> PagingResponse[Device]:
+        pass
+
+    all_devices = collect(devices)
+
+    @get("dcim/devices/{device_id}")
+    def get_device(
+            self,
+            device_id: int,
+    ) -> Device:
+        pass