annet 0.16.8__py3-none-any.whl → 0.16.10__py3-none-any.whl

annet/bgp_models.py

@@ -0,0 +1,266 @@
+from dataclasses import dataclass, field
+from typing import Literal, Union, Optional
+
+
+class ASN(int):
+    """
+    Stores ASN number and formats it as в AS1.AS2
+    None is treated like 0. Supports integer operations
+    Supported formats: https://tools.ietf.org/html/rfc5396#section-1
+    """
+    PLAIN_MAX = 0x10000
+
+    def __new__(cls, asn: Union[int, str, None, "ASN"]):
+        if isinstance(asn, ASN):
+            return asn
+        elif asn is None:
+            asn = 0
+        elif not isinstance(asn, int):
+            if isinstance(asn, str) and "." in asn:
+                high, low = [int(token) for token in asn.split(".")]
+                if not (0 <= high < ASN.PLAIN_MAX and 0 <= low < ASN.PLAIN_MAX):
+                    raise ValueError("Invalid ASN asn %r" % asn)
+                asn = (high << 16) + low
+            asn = int(asn)
+        if not 0 <= asn <= 0xffffffff:
+            raise ValueError("Invalid ASN asn %r" % asn)
+        return int.__new__(cls, asn)
+
+    def __add__(self, other: int):
+        return ASN(super().__add__(other))
+
+    def __sub__(self, other: int):
+        return ASN(super().__sub__(other))
+
+    def __mul__(self, other: int):
+        return ASN(super().__mul__(other))
+
+    def is_plain(self) -> bool:
+        return self < ASN.PLAIN_MAX
+
+    def asdot(self) -> str:
+        if not self.is_plain():
+            return "%d.%d" % (self // ASN.PLAIN_MAX, self % ASN.PLAIN_MAX)
+        return "%d" % self
+
+    def asplain(self) -> str:
+        return "%d" % self
+
+    def asdot_high(self) -> int:
+        return self // ASN.PLAIN_MAX
+
+    def asdot_low(self) -> int:
+        return self % ASN.PLAIN_MAX
+
+    __str__ = asdot
+
+    def __repr__(self) -> str:
+        srepr = str(self)
+        if "." in srepr:
+            srepr = repr(srepr)
+        return f"{self.__class__.__name__}({srepr})"
+
+
+@dataclass(frozen=True)
+class BFDTimers:
+    minimum_interval: int = 500
+    multiplier: int = 4
+
+
+Family = Literal["ipv4_unicast", "ipv6_unicast", "ipv4_labeled", "ipv6_labeled"]
+
+
+@dataclass(frozen=True)
+class PeerOptions:
+    """The same options as for group but any field is optional"""
+    local_as: Optional[ASN] = None
+    unnumbered: Optional[bool] = None
+    rr_client: Optional[bool] = None
+    next_hop_self: Optional[bool] = None
+    extended_next_hop: Optional[bool] = None
+    send_community: Optional[bool] = None
+    send_lcommunity: Optional[bool] = None
+    send_extcommunity: Optional[bool] = None
+    send_labeled: Optional[bool] = None
+    import_limit: Optional[bool] = None
+    teardown_timeout: Optional[bool] = None
+    redistribute: Optional[bool] = None
+    passive: Optional[bool] = None
+    mtu_discovery: Optional[bool] = None
+    advertise_inactive: Optional[bool] = None
+    advertise_bgp_static: Optional[bool] = None
+    allowas_in: Optional[bool] = None
+    auth_key: Optional[bool] = None
+    add_path: Optional[bool] = None
+    multipath: Optional[bool] = None
+    multihop: Optional[bool] = None
+    multihop_no_nexthop_change: Optional[bool] = None
+    af_no_install: Optional[bool] = None
+    bfd: Optional[bool] = None
+    rib: Optional[bool] = None
+    bfd_timers: Optional[BFDTimers] = None
+    resolve_vpn: Optional[bool] = None
+    af_rib_group: Optional[str] = None
+    af_loops: Optional[int] = None
+    hold_time: Optional[int] = None
+    listen_network: Optional[bool] = None
+    remove_private: Optional[bool] = None
+    as_override: Optional[bool] = None
+    aigp: Optional[bool] = None
+    bmp_monitor: Optional[bool] = None
+    no_prepend: Optional[bool] = None
+    no_explicit_null: Optional[bool] = None
+    uniq_iface: Optional[bool] = None
+    advertise_peer_as: Optional[bool] = None
+    connect_retry: Optional[bool] = None
+    advertise_external: Optional[bool] = None
+    advertise_irb: Optional[bool] = None
+    listen_only: Optional[bool] = None
+    soft_reconfiguration_inbound: Optional[bool] = None
+    not_active: Optional[bool] = None
+    mtu: Optional[int] = None
+
+
+@dataclass
+class Peer:
+    addr: str
+    interface: Optional[str]
+    remote_as: ASN
+    families: set[Family] = field(default_factory=set)
+    description: str = ""
+    vrf_name: str = ""
+    group_name: str = ""
+    import_policy: str = ""
+    export_policy: str = ""
+    update_source: Optional[str] = None
+    options: Optional[PeerOptions] = None
+    hostname: str = ""
+
+
+@dataclass
+class Aggregate:
+    policy: str = ""
+    routes: tuple[str, ...] = ()  # "182.168.1.0/24",
+    export_policy: str = ""
+    as_path: str = ""
+    reference: str = ""
+    suppress: bool = False
+    discard: bool = True
+    as_set: bool = False
+
+
+@dataclass
+class Redistribute:
+    protocol: str
+    policy: str = ""
+
+
+@dataclass
+class FamilyOptions:
+    family: Family
+    vrf_name: str
+    multipath: int = 0
+    global_multipath: int = 0
+    aggregate: Aggregate = field(default_factory=Aggregate)
+    redistributes: tuple[Redistribute, ...] = ()
+    allow_default: bool = False
+    aspath_relax: bool = False
+    igp_ignore: bool = False
+    next_hop_policy: bool = False
+    rib_import_policy: bool = False
+    advertise_l2vpn_evpn: bool = False
+    rib_group: bool = False
+    loops: int = 0
+    advertise_bgp_static: bool = False
+
+
+@dataclass(frozen=True)
+class PeerGroup:
+    name: str
+    remote_as: ASN = ASN(None)
+    internal_name: str = ""
+    description: str = ""
+    update_source: str = ""
+
+    # more strict version of PeerOptions
+    local_as: ASN = ASN(None)
+    unnumbered: bool = False
+    rr_client: bool = False
+    next_hop_self: bool = False
+    extended_next_hop: bool = False
+    send_community: bool = False
+    send_lcommunity: bool = False
+    send_extcommunity: bool = False
+    send_labeled: bool = False
+    import_limit: bool = False
+    teardown_timeout: bool = False
+    redistribute: bool = False
+    passive: bool = False
+    mtu_discovery: bool = False
+    advertise_inactive: bool = False
+    advertise_bgp_static: bool = False
+    allowas_in: bool = False
+    auth_key: bool = False
+    add_path: bool = False
+    multipath: bool = False
+    multihop: bool = False
+    multihop_no_nexthop_change: bool = False
+    af_no_install: bool = False
+    bfd: bool = False
+    rib: bool = False
+    bfd_timers: Optional[BFDTimers] = None
+    resolve_vpn: bool = False
+    af_rib_group: Optional[str] = None
+    af_loops: int = 0
+    hold_time: int = 0
+    listen_network: bool = False
+    remove_private: bool = False
+    as_override: bool = False
+    aigp: bool = False
+    bmp_monitor: bool = False
+    no_prepend: bool = False
+    no_explicit_null: bool = False
+    uniq_iface: bool = False
+    advertise_peer_as: bool = False
+    connect_retry: bool = False
+    advertise_external: bool = False
+    advertise_irb: bool = False
+    listen_only: bool = False
+    soft_reconfiguration_inbound: bool = False
+    not_active: bool = False
+    mtu: int = 0
+
+
+@dataclass
+class VrfOptions:
+    vrf_name: str
+    vrf_name_global: Optional[str] = None
+    import_policy: Optional[str] = None
+    export_policy: Optional[str] = None
+    rt_import: list[str] = field(default_factory=list)
+    rt_export: list[str] = field(default_factory=list)
+    rt_import_v4: list[str] = field(default_factory=list)
+    rt_export_v4: list[str] = field(default_factory=list)
+    route_distinguisher: Optional[str] = None
+    static_label: Optional[int] = None  # FIXME: str?
+
+    ipv4_unicast: Optional[FamilyOptions] = None
+    ipv6_unicast: Optional[FamilyOptions] = None
+    ipv4_labeled_unicast: Optional[FamilyOptions] = None
+    ipv6_labeled_unicast: Optional[FamilyOptions] = None
+    groups: list[PeerGroup] = field(default_factory=list)
+
+
+@dataclass
+class GlobalOptions:
+    local_as: ASN = ASN(None)
+    loops: int = 0
+    multipath: int = 0
+    router_id: str = ""
+    vrf: dict[str, VrfOptions] = field(default_factory=dict)
+
+    ipv4_unicast: Optional[FamilyOptions] = None
+    ipv6_unicast: Optional[FamilyOptions] = None
+    ipv4_labeled_unicast: Optional[FamilyOptions] = None
+    ipv6_labeled_unicast: Optional[FamilyOptions] = None
+    groups: list[PeerGroup] = field(default_factory=list)

annet/configs/context.yml

@@ -1,24 +1,14 @@
-connection:
-  default:
-    login: ~
-    passwords: ~
-    enable_ssh_conf: false
-    no_nocauth: false
-    ssh_forward_agent: ~
-    tunnel: ~
 generators:
   default:
     - annet_generators.example
+    - annet_generators.mesh_example
 
 storage:
   default:
-    adapter: file
-    params:
-      path: /path/to/file
+    adapter: netbox
 
 context:
   default:
-    connection: default
     generators: default
     storage: default
 

annet/configs/logging.yaml

@@ -7,6 +7,7 @@ formatters:
         style: "{"
         datefmt: "%H:%M:%S"
         format: "[{asctime}] {log_color}{levelname:>7}{reset} {processName} - {pathname}:{lineno} - {message} -- {cyan}{_extra}{reset}"
+        max_vars_lines: 20
     host_progress:
         (): contextlog.SmartFormatter
         style: "{"

annet/generators/__init__.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import dataclasses
 import importlib
+import importlib.machinery
 import os
+import re
 import textwrap
 from collections import OrderedDict as odict
 from typing import (
@@ -379,7 +381,7 @@ def _run_json_fragment_generator(
 def _get_generators(module_paths: Union[List[str], dict], storage, device=None):
     if isinstance(module_paths, dict):
         if device is None:
-            module_paths = module_paths.get("default")
+            module_paths = module_paths.get("default", [])
         else:
             modules = []
             seen = set()
@@ -396,22 +398,34 @@ def _get_generators(module_paths: Union[List[str], dict], storage, device=None):
                         if module not in seen:
                             modules.append(module)
                             seen.add(module)
-            module_paths = modules or module_paths.get("default")
+            module_paths = modules or module_paths.get("default", [])
     res_generators = []
     for module_path in module_paths:
-        module = importlib.import_module(module_path)
+        module = _load_gen_module(module_path)
         if hasattr(module, "get_generators"):
             generators: List[BaseGenerator] = module.get_generators(storage)
             res_generators += generators
     return res_generators
 
 
+def _load_gen_module(module_path: str):
+    try:
+        module = importlib.import_module(module_path)
+    except ModuleNotFoundError as e:
+        try:  # maybe it's a path to module
+            module_abs_path = os.path.abspath(module_path)
+            module = importlib.machinery.SourceFileLoader(re.sub(r"[./]", "_", module_abs_path).strip("_"), module_abs_path).load_module()
+        except ModuleNotFoundError:
+            raise e
+    return module
+
+
 def _get_ref_generators(module_paths: List[str], storage, device):
     if isinstance(module_paths, dict):
         module_paths = module_paths.get("default")
     res_generators = []
     for module_path in module_paths:
-        module = importlib.import_module(module_path)
+        module = _load_gen_module(module_path)
         if hasattr(module, "get_ref_generators"):
             res_generators += module.get_ref_generators(storage)
     return res_generators

annet/generators/jsonfragment.py

@@ -86,9 +86,12 @@ class JSONFragment(TreeGenerator):
             self._config_pointer.pop()
 
     def __call__(self, device: Device, annotate: bool = False):
-        for cfg_fragment in self.run(device):
-            self._set_or_replace_dict(self._config_pointer, cfg_fragment)
-        return self._json_config
+        try:
+            for cfg_fragment in self.run(device):
+                self._set_or_replace_dict(self._config_pointer, cfg_fragment)
+            return self._json_config
+        finally:
+            self._json_config = {}
 
     def _set_or_replace_dict(self, pointer, value):
         if not pointer:
@@ -99,27 +102,25 @@ class JSONFragment(TreeGenerator):
         else:
             self._set_dict(self._json_config, pointer, value)
 
-    @classmethod
-    def _to_str(cls, value: Any) -> str:
+    def process_value(self, value: Any) -> Any:
         if isinstance(value, str):
             return value
         elif isinstance(value, list):
-            return [cls._to_str(x) for x in value]
+            return [self.process_value(x) for x in value]
         elif isinstance(value, dict):
             for k, v in value.items():
-                value[k] = cls._to_str(v)
+                value[k] = self.process_value(v)
             return value
         return str(value)
 
-    @classmethod
-    def _set_dict(cls, cfg, pointer, value):
+    def _set_dict(self, cfg, pointer, value):
         # pointer has at least one key
         if len(pointer) == 1:
             if pointer[0] in cfg:
-                cfg[pointer[0]] = [cfg[pointer[0]], cls._to_str(value)]
+                cfg[pointer[0]] = [cfg[pointer[0]], self.process_value(value)]
             else:
-                cfg[pointer[0]] = cls._to_str(value)
+                cfg[pointer[0]] = self.process_value(value)
         else:
             if pointer[0] not in cfg:
                 cfg[pointer[0]] = {}
-            cls._set_dict(cfg[pointer[0]], pointer[1:], cls._to_str(value))
+            self._set_dict(cfg[pointer[0]], pointer[1:], self.process_value(value))

annet/mesh/__init__.py

@@ -0,0 +1,16 @@
+__all__ = [
+    "GlobalOptions",
+    "MeshSession",
+    "DirectPeer",
+    "IndirectPeer",
+    "MeshExecutor",
+    "MeshRulesRegistry",
+    "Left",
+    "Right",
+    "Match",
+]
+
+from .executor import MeshExecutor
+from .match_args import Left, Right, Match
+from .registry import DirectPeer, IndirectPeer, MeshSession, GlobalOptions
+from .registry import MeshRulesRegistry

annet/mesh/basemodel.py

@@ -0,0 +1,180 @@
+from abc import ABC
+from copy import copy
+from enum import Enum
+from typing import (
+    TypeVar, Any, Annotated, get_origin, get_type_hints, get_args, Callable, Union, ClassVar, overload,
+)
+
+
+class MergeForbiddenError(Exception):
+    pass
+
+
+class Special(Enum):
+    NOT_SET = "<NOT SET>"
+
+
+T = TypeVar("T")
+
+
+class Merger(ABC):
+    def __call__(self, name: str, x: Union[T, Special], y: Union[T, Special]) -> Union[T, Special]:
+        if x is Special.NOT_SET:
+            return y
+        if y is Special.NOT_SET:
+            return x
+        return self._merge(name, x, y)
+
+    def _merge(self, name: str, x: T, y: T) -> T:
+        raise NotImplementedError
+
+
+class UseFirst(Merger):
+    def _merge(self, name: str, x: T, y: T) -> T:
+        return x
+
+
+class UseLast(Merger):
+    def _merge(self, name: str, x: T, y: T) -> T:
+        return y
+
+
+class Forbid(Merger):
+    def _merge(self, name: str, x: T, y: T) -> T:
+        raise MergeForbiddenError(f"Override is forbidden for field {name}")
+
+
+class ForbidChange(Merger):
+    def _merge(self, name: str, x: T, y: T) -> T:
+        if x == y:
+            return x
+        raise MergeForbiddenError(
+            f"Override with different value is forbidden for field {name}:\n"
+            f"Old: {x}\n"
+            f"New: {y}"
+        )
+
+
+class Concat(Merger):
+    def _merge(self, name: str, x: T, y: T) -> T:
+        return x + y  # type: ignore[operator]
+
+
+class Merge(Merger):
+    def _merge(self, name: str, x: "ModelT", y: "ModelT") -> "ModelT":  # type: ignore[override]
+        return merge(x, y)
+
+
+class DictMerge(Merger):
+    def __init__(self, value_merger: Merger = Forbid()):
+        self.value_merger = value_merger
+
+    def _merge(self, name: str, x: dict, y: dict) -> dict:  # type: ignore[override]
+        result = copy(x)
+        for key, value in y.items():
+            if key in result:
+                result[key] = self.value_merger(key, result[key], value)
+            else:
+                result[key] = value
+        return result
+
+
+class ApplyFunc(Merger):
+    def __init__(self, func: Callable):
+        self.func = func
+
+    def __call__(self, name: str, x: Union[T, Special], y: Union[T, Special]) -> Union[T, Special]:
+        if x is Special.NOT_SET:
+            return y
+        if y is Special.NOT_SET:
+            return x
+        return self.func(x, y)
+
+
+def _get_merger(hint: Any):
+    if get_origin(hint) is not Annotated:
+        return ForbidChange()
+    for arg in get_args(hint):
+        if isinstance(arg, Merger):
+            return arg
+    return ForbidChange()
+
+
+class BaseMeshModel:
+    _field_mergers: ClassVar[dict[str, Merger]]
+
+    def __init__(self, **kwargs):
+        if extra_keys := (kwargs.keys() - self._field_mergers.keys()):
+            raise ValueError(f"Extra arguments: {extra_keys}")
+        self.__dict__.update(kwargs)
+
+    def unset_attrs(self):
+        return type(self)._field_mergers.keys() - vars(self).keys()
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(" + ", ".join(
+            f"{key}={value}" for key, value in vars(self).items()
+        ) + ")"
+
+    def __init_subclass__(cls, **kwargs):
+        cls._field_mergers = {
+            field: _get_merger(hint)
+            for field, hint in get_type_hints(cls, include_extras=True).items()
+            if get_origin(hint) is not ClassVar
+        }
+
+    def __setattr__(self, key, value):
+        if key not in self._field_mergers:
+            raise AttributeError(f"{self.__class__.__name__} has no field {key}")
+        super().__setattr__(key, value)
+
+
+ModelT = TypeVar("ModelT", bound=BaseMeshModel)
+
+
+def _merge(a: ModelT, b: BaseMeshModel) -> ModelT:
+    result = copy(a)
+    for attr_name, merger in a._field_mergers.items():
+        new_value = merger(
+            attr_name,
+            getattr(a, attr_name, Special.NOT_SET),
+            getattr(b, attr_name, Special.NOT_SET),
+        )
+        if new_value is not Special.NOT_SET:
+            setattr(result, attr_name, new_value)
+    return result
+
+
+@overload
+def merge(first: Special, second: ModelT, /, *others: BaseMeshModel) -> ModelT:
+    ...
+
+
+@overload
+def merge(first: ModelT, /, *others: BaseMeshModel) -> ModelT:
+    ...
+
+
+@overload
+def merge(first: Special, /) -> Special:
+    ...
+
+
+def merge(first: Any, /, *others: Any) -> Any:
+    if first is Special.NOT_SET:
+        if not others:
+            return Special.NOT_SET
+        return merge(*others)
+    for second in others:
+        first = _merge(first, second)
+    return first
+
+
+class KeyDefaultDict(dict):
+    def __init__(self, factory: Callable[[str], Any]):
+        super().__init__()
+        self.factory = factory
+
+    def __missing__(self, key):
+        x = self[key] = self.factory(key)
+        return x

annet/mesh/device_models.py

@@ -0,0 +1,62 @@
+from typing import Annotated, Optional, Union
+
+from annet.bgp_models import Family, Aggregate, Redistribute
+from .basemodel import BaseMeshModel, Concat, DictMerge, Merge, KeyDefaultDict
+from .peer_models import MeshPeerGroup
+
+
+class FamilyOptions(BaseMeshModel):
+    family: Family
+    vrf_name: str
+    multipath: int = 0
+    global_multipath: int
+    aggregate: Aggregate
+    redistributes: Annotated[tuple[Redistribute, ...], Concat()]
+    allow_default: bool
+    aspath_relax: bool
+    igp_ignore: bool
+    next_hop_policy: bool
+    rib_import_policy: bool
+    advertise_l2vpn_evpn: bool
+    rib_group: bool
+    loops: int
+    advertise_bgp_static: bool
+
+
+class _FamiliesMixin:
+    ipv4_unicast: Optional[FamilyOptions]
+    ipv6_unicast: Optional[FamilyOptions]
+    ipv4_labeled_unicast: Optional[FamilyOptions]
+    ipv6_labeled_unicast: Optional[FamilyOptions]
+
+
+class VrfOptions(BaseMeshModel, _FamiliesMixin):
+    def __init__(self, **kwargs):
+        kwargs.setdefault("groups", KeyDefaultDict(lambda x: MeshPeerGroup(name=x)))
+        super().__init__(**kwargs)
+
+    vrf_name: str
+    vrf_name_global: Optional[str]
+    import_policy: Optional[str]
+    export_policy: Optional[str]
+    rt_import: Annotated[tuple[str, ...], Concat()]
+    rt_export: Annotated[tuple[str, ...], Concat()]
+    rt_import_v4: Annotated[tuple[str, ...], Concat()]
+    rt_export_v4: Annotated[tuple[str, ...], Concat()]
+    route_distinguisher: Optional[str]
+    static_label: Optional[int]  # FIXME: str?
+    groups: Annotated[dict[str, MeshPeerGroup], DictMerge(Merge())]
+
+
+class GlobalOptionsDTO(BaseMeshModel, _FamiliesMixin):
+    def __init__(self, **kwargs):
+        kwargs.setdefault("groups", KeyDefaultDict(lambda x: MeshPeerGroup(name=x)))
+        kwargs.setdefault("vrf", KeyDefaultDict(lambda x: VrfOptions(vrf_name=x)))
+        super().__init__(**kwargs)
+
+    local_as: Union[int, str]
+    loops: int
+    multipath: int
+    router_id: str
+    vrf: Annotated[dict[str, VrfOptions], DictMerge(Merge())]
+    groups: Annotated[dict[str, MeshPeerGroup], DictMerge(Merge())]