annet 0.16.7__py3-none-any.whl → 0.16.9__py3-none-any.whl

annet/mesh/executor.py

@@ -0,0 +1,248 @@
+from dataclasses import dataclass
+from logging import getLogger
+from typing import Annotated, Callable, Optional
+
+from annet.bgp_models import Peer, GlobalOptions
+from annet.storage import Device, Storage
+from .basemodel import merge, BaseMeshModel, Merge, UseLast, MergeForbiddenError
+from .device_models import GlobalOptionsDTO
+from .models_converter import to_bgp_global_options, to_bgp_peer, InterfaceChanges, to_interface_changes
+from .peer_models import PeerDTO
+from .registry import MeshRulesRegistry, GlobalOptions as MeshGlobalOptions, DirectPeer, MeshSession, IndirectPeer
+
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class MeshExecutionResult:
+    global_options: GlobalOptions
+    peers: list[Peer]
+
+
+@dataclass(frozen=True)
+class PeerKey:
+    fqdn: str
+    addr: str
+    vrf: str
+
+
+class Pair(BaseMeshModel):
+    local: Annotated[PeerDTO, Merge()]
+    connected: Annotated[PeerDTO, Merge()]
+    device: Annotated[Device, UseLast()]
+
+
+class MeshExecutor:
+    def __init__(
+            self,
+            registry: MeshRulesRegistry,
+            storage: Storage,
+    ):
+        self._registry = registry
+        self._storage = storage
+
+    def _execute_globals(self, device: Device) -> GlobalOptionsDTO:
+        global_opts = GlobalOptionsDTO()
+        for rule in self._registry.lookup_global(device.fqdn):
+            handler_name = self._handler_name(rule.handler)
+            rule_global_opts = MeshGlobalOptions(rule.match, device)
+            logger.debug("Running device handler: %s", handler_name)
+            rule.handler(rule_global_opts)
+            try:
+                global_opts = merge(global_opts, rule_global_opts)
+            except MergeForbiddenError as e:
+                raise ValueError(
+                    f"Handler `{handler_name}` global options conflicting with "
+                    f"previously loaded for device `{device.fqdn}`:\n" + str(e)
+                ) from e
+        return global_opts
+
+    def _handler_name(self, handler: Callable) -> str:
+        try:
+            return f"{handler.__module__}.{handler.__qualname__}"
+        except AttributeError:
+            return str(handler)
+
+    def _execute_direct(self, device: Device) -> list[Pair]:
+        # we can have multiple rules for the same pair
+        # we merge them according to remote fqdn
+        neighbor_peers: dict[PeerKey, Pair] = {}
+        # TODO batch resolve
+        for rule in self._registry.lookup_direct(device.fqdn, device.neighbours_fqdns):
+            session = MeshSession()
+            handler_name = self._handler_name(rule.handler)
+            logger.debug("Running direct handler: %s", handler_name)
+            if rule.direct_order:
+                neighbor_device = self._storage.make_devices([rule.name_right])[0]
+                peer_device = DirectPeer(rule.match_left, device, [])
+                peer_neighbor = DirectPeer(rule.match_right, neighbor_device, [])
+            else:
+                neighbor_device = self._storage.make_devices([rule.name_left])[0]
+                peer_neighbor = DirectPeer(rule.match_left, neighbor_device, [])
+                peer_device = DirectPeer(rule.match_right, device, [])
+
+            interfaces = self._storage.search_connections(device, neighbor_device)
+            for local_port, remote_port in interfaces:
+                peer_device.ports.append(local_port.name)
+                peer_neighbor.ports.append(remote_port.name)
+
+            if rule.direct_order:
+                rule.handler(peer_device, peer_neighbor, session)
+            else:
+                rule.handler(peer_neighbor, peer_device, session)
+
+            try:
+                neighbor_dto = merge(PeerDTO(), peer_neighbor, session)
+            except MergeForbiddenError as e:
+                raise ValueError(
+                    f"Handler `{handler_name}` provided session data conflicting with "
+                    f"peer data for device `{neighbor_device.fqdn}`:\n" + str(e)
+                ) from e
+            try:
+                device_dto = merge(PeerDTO(), peer_device, session)
+            except MergeForbiddenError as e:
+                raise ValueError(
+                    f"Handler `{handler_name}` provided session data conflicting with "
+                    f"peer data for device `{device.fqdn}`:\n" + str(e)
+                ) from e
+
+            pair = Pair(local=device_dto, connected=neighbor_dto, device=neighbor_device)
+            addr = getattr(neighbor_dto, "addr", None)
+            if addr is None:
+                raise ValueError(f"Handler `{handler_name}` returned no peer addr")
+            peer_key = PeerKey(
+                fqdn=neighbor_device.fqdn,
+                addr=addr,
+                vrf=getattr(neighbor_dto, "vrf", "")
+            )
+            try:
+                if peer_key in neighbor_peers:
+                    pair = merge(neighbor_peers[peer_key], pair)
+            except MergeForbiddenError as e:
+                if rule.direct_order:
+                    pair_names = device.fqdn, neighbor_device.fqdn
+                else:
+                    pair_names = neighbor_device.fqdn, device.fqdn
+                raise ValueError(
+                    f"Handler `{handler_name}` provides data conflicting with "
+                    f"previously loaded for device pair {pair_names} "
+                    f"with addr={peer_key.addr}, vrf{peer_key.vrf}:\n" + str(e)
+                ) from e
+            neighbor_peers[peer_key] = pair
+        return list(neighbor_peers.values())
+
+    def _execute_indirect(self, device: Device, all_fqdns: list[str]) -> list[Pair]:
+        # we can have multiple rules for the same pair
+        # we merge them according to remote fqdn
+        connected_peers: dict[PeerKey, Pair] = {}
+        for rule in self._registry.lookup_indirect(device.fqdn, all_fqdns):
+            session = MeshSession()
+            handler_name = self._handler_name(rule.handler)
+            logger.debug("Running indirect handler: %s", handler_name)
+            if rule.direct_order:
+                connected_device = self._storage.make_devices([rule.name_right])[0]
+                peer_device = IndirectPeer(rule.match_left, device)
+                peer_connected = IndirectPeer(rule.match_right, connected_device)
+                rule.handler(peer_device, peer_connected, session)
+            else:
+                connected_device = self._storage.make_devices([rule.name_left])[0]
+                peer_connected = IndirectPeer(rule.match_left, connected_device)
+                peer_device = IndirectPeer(rule.match_right, device)
+                rule.handler(peer_connected, peer_device, session)
+
+            try:
+                connected_dto = merge(PeerDTO(), peer_connected, session)
+            except MergeForbiddenError as e:
+                raise ValueError(
+                    f"Handler `{handler_name}` provided session data conflicting with "
+                    f"peer data for device `{connected_device.fqdn}`:\n" + str(e)
+                ) from e
+            try:
+                device_dto = merge(PeerDTO(), peer_device, session)
+            except MergeForbiddenError as e:
+                raise ValueError(
+                    f"Handler `{handler_name}` provided session data conflicting with "
+                    f"peer data for device `{device.fqdn}`:\n" + str(e)
+                ) from e
+
+            pair = Pair(local=device_dto, connected=connected_dto, device=connected_device)
+            addr = getattr(connected_dto, "addr", None)
+            if addr is None:
+                raise ValueError(f"Handler `{handler_name}` returned no peer addr")
+            peer_key = PeerKey(
+                fqdn=connected_device.fqdn,
+                addr=addr,
+                vrf=getattr(connected_dto, "vrf", "")
+            )
+            try:
+                if peer_key in connected_peers:
+                    pair = merge(connected_peers[peer_key], pair)
+            except MergeForbiddenError as e:
+                if rule.direct_order:
+                    pair_names = device.fqdn, connected_device.fqdn
+                else:
+                    pair_names = connected_device.fqdn, device.fqdn
+                raise ValueError(
+                    f"Handler `{handler_name}` provides data conflicting with "
+                    f"previously loaded for device pair {pair_names} "
+                    f"with addr={peer_key.addr}, vrf{peer_key.vrf}:\n" + str(e)
+                ) from e
+            connected_peers[peer_key] = pair
+
+        return list(connected_peers.values())  # FIXME
+
+    def _to_bgp_peer(self, pair: Pair, interface: Optional[str]) -> Peer:
+        return to_bgp_peer(pair.local, pair.connected, pair.device, interface)
+
+    def _to_bgp_global(self, global_options: GlobalOptionsDTO) -> GlobalOptions:
+        return to_bgp_global_options(global_options)
+
+    def _apply_interface_changes(self, device: Device, neighbor: Device, changes: InterfaceChanges) -> str:
+        port_pairs = self._storage.search_connections(device, neighbor)
+        if len(port_pairs) > 1:
+            if changes.lag is changes.svi is None:
+                raise ValueError(
+                    f"Multiple connections found between {device.fqdn} and {neighbor.fqdn}."
+                    "Specify LAG or SVI"
+                )
+        if changes.lag is not None:
+            target_interface = device.make_lag(
+                lag=changes.lag,
+                ports=[local_port.name for local_port, remote_port in port_pairs],
+                lag_min_links=changes.lag_links_min,
+            )
+            if changes.subif is not None:
+                target_interface = device.add_subif(target_interface.name, changes.subif)
+        elif changes.subif is not None:
+            # single connection
+            local_port, remote_port = port_pairs[0]
+            target_interface = device.add_subif(local_port.name, changes.subif)
+        elif changes.svi is not None:
+            target_interface = device.add_svi(changes.svi)
+        else:
+            target_interface, _ = port_pairs[0]
+        target_interface.add_addr(changes.addr, changes.vrf)
+        return target_interface.name
+
+    def execute_for(self, device: Device) -> MeshExecutionResult:
+        all_fqdns = self._storage.resolve_all_fdnds()
+
+        global_options = self._to_bgp_global(self._execute_globals(device))
+
+        peers = []
+        for direct_pair in self._execute_direct(device):
+            target_interface = self._apply_interface_changes(
+                device,
+                direct_pair.device,
+                to_interface_changes(direct_pair.local),
+            )
+            peers.append(self._to_bgp_peer(direct_pair, target_interface))
+
+        for connected_pair in self._execute_indirect(device, all_fqdns):
+            peers.append(self._to_bgp_peer(connected_pair, None))
+
+        return MeshExecutionResult(
+            global_options=global_options,
+            peers=peers,
+        )

annet/mesh/match_args.py

@@ -0,0 +1,165 @@
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from types import SimpleNamespace
+from typing import Any, Sequence, Optional
+
+MatchedArgs = SimpleNamespace
+
+
+def identity(x: Any) -> Any:
+    return x
+
+
+class MatchExpr:
+    def __init__(self, expr: Callable[[Any], Any] = identity):
+        self.expr = expr
+
+    def __getattr__(self, item: str):
+        return MatchExpr(lambda x: getattr(self.expr(x), item))
+
+    def __getitem__(self, item: Any):
+        return MatchExpr(lambda x: self.expr(x)[item])
+
+    def __eq__(self, other) -> "MatchExpr":  # type: ignore[override]  # https://github.com/python/mypy/issues/5951
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) == other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) == other)
+
+    def __ne__(self, other) -> "MatchExpr":  # type: ignore[override]  # https://github.com/python/mypy/issues/5951
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) != other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) != other)
+
+    def __lt__(self, other) -> "MatchExpr":
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) < other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) < other)
+
+    def __gt__(self, other) -> "MatchExpr":
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) > other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) > other)
+
+    def __le__(self, other) -> "MatchExpr":
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) <= other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) <= other)
+
+    def __ge__(self, other) -> "MatchExpr":
+        if isinstance(other, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) >= other.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) >= other)
+
+    def __or__(self, other: "MatchExpr") -> "MatchExpr":
+        return MatchExpr(lambda x: self.expr(x) or other.expr(x))
+
+    def __and__(self, other: "MatchExpr") -> "MatchExpr":
+        return MatchExpr(lambda x: self.expr(x) and other.expr(x))
+
+    def cast_(self, type_: Callable[[Any], Any]) -> "MatchExpr":
+        return MatchExpr(lambda x: type_(self.expr(x)))
+
+    def in_(self, value: Any) -> "MatchExpr":
+        if isinstance(value, MatchExpr):
+            return MatchExpr(lambda x: self.expr(x) in value.expr(x))
+        else:
+            return MatchExpr(lambda x: self.expr(x) in value)
+
+
+Match = MatchExpr()[0]
+Left = MatchExpr()[0]
+Right = MatchExpr()[1]
+
+
+class PeerNameTemplate:
+    def __init__(self, raw_str):
+        self._str = str(raw_str)
+        self._regex, self._types = self._compile(self._str)
+
+    def __str__(self):
+        return self._str
+
+    @staticmethod
+    def _compile(value: str) -> tuple[re.Pattern[str], dict[str, type]]:
+        int_groups = re.findall(r"{(?P<group_name>\w+)}", value)
+        # '{name}'  -> (?P<name>\d+)
+        regex_string = re.sub(r"{(?P<group_name>\w+)}", r"(?P<\g<group_name>>\\d+)", value)
+        # '{name:regex}' -> (?P<name>regex)
+        regex_string = re.sub(
+            r"{(?P<group_name>\w+):(?P<custom_regex>.*?)}", r"(?P<\g<group_name>>\g<custom_regex>)",
+            regex_string
+        )
+        pattern = re.compile(regex_string)
+        types: dict[str, type] = {
+            name: (int if name in int_groups else str)
+            for name in pattern.groupindex
+        }
+        return pattern, types
+
+    def match(self, hostname: str) -> Optional[dict[str, str]]:
+        reg_match = self._regex.fullmatch(hostname)
+        if reg_match:
+            return {
+                key: self._types[key](value)
+                for key, value in reg_match.groupdict().items()
+            }
+        return None
+
+
+def match_safe(match_expressions: Sequence[MatchExpr], value: Any) -> bool:
+    for matcher in match_expressions:
+        try:
+            res = matcher.expr(value)
+            if not bool(res):
+                return False
+        except (TypeError, ValueError, AttributeError, KeyError, IndexError):
+            return False
+    return True
+
+
+@dataclass
+class SingleMatcher:
+    def __init__(self, rule: str, match_expressions: Sequence[MatchExpr]):
+        self.rule = PeerNameTemplate(rule)
+        self.match_expressions = match_expressions
+
+    def match_one(self, host: str) -> Optional[MatchedArgs]:
+        data = self.rule.match(host)
+        if data is None:
+            return None
+        args = MatchedArgs(**data)
+        if not match_safe(self.match_expressions, (args,)):
+            return None
+        return args
+
+
+@dataclass
+class PairMatcher:
+    def __init__(self, left_rule: str, right_rule: str, match_expressions: Sequence[MatchExpr]):
+        self.left_rule = PeerNameTemplate(left_rule)
+        self.right_rule = PeerNameTemplate(right_rule)
+        self.match_expressions = match_expressions
+
+    def match_pair(self, left: str, right: str) -> Optional[tuple[MatchedArgs, MatchedArgs]]:
+        left_args = self._match_host(self.left_rule, left)
+        if left_args is None:
+            return None
+        right_args = self._match_host(self.right_rule, right)
+        if right_args is None:
+            return None
+        if not match_safe(self.match_expressions, (left_args, right_args)):
+            return None
+        return left_args, right_args
+
+    def _match_host(self, rule: PeerNameTemplate, host: str) -> Optional[MatchedArgs]:
+        data = rule.match(host)
+        if data is None:
+            return None
+        return MatchedArgs(**data)

annet/mesh/models_converter.py

@@ -0,0 +1,84 @@
+from dataclasses import dataclass
+from ipaddress import ip_interface
+from typing import Optional
+
+from adaptix import Retort, loader, Chain, name_mapping
+
+from .peer_models import PeerDTO
+from ..bgp_models import GlobalOptions, VrfOptions, FamilyOptions, Peer, PeerGroup, ASN, PeerOptions
+from ..storage import Device
+
+
+@dataclass
+class InterfaceChanges:
+    addr: str
+    lag: Optional[int] = None
+    lag_links_min: Optional[int] = None
+    svi: Optional[int] = None
+    subif: Optional[int] = None
+    vrf: Optional[str] = None
+
+    def __post_init__(self):
+        if self.lag is not None and self.svi is not None:
+            raise ValueError("Cannot use LAG and SVI together")
+        if self.svi is not None and self.subif is not None:
+            raise ValueError("Cannot use Subif and SVI together")
+
+
+class ObjMapping:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __contains__(self, item):
+        return hasattr(self.obj, item)
+
+    def __getitem__(self, name):
+        return getattr(self.obj, name)
+
+    def get(self, name, default=None):
+        return getattr(self.obj, name, default)
+
+
+retort = Retort(
+    recipe=[
+        loader(InterfaceChanges, ObjMapping, Chain.FIRST),
+        loader(ASN, ASN),
+        loader(GlobalOptions, ObjMapping, Chain.FIRST),
+        loader(VrfOptions, ObjMapping, Chain.FIRST),
+        loader(FamilyOptions, ObjMapping, Chain.FIRST),
+        loader(PeerOptions, ObjMapping, Chain.FIRST),
+        name_mapping(PeerOptions, map={
+            "local_as": "asnum",
+        }),
+        loader(list[PeerGroup], lambda x: list(x.values()), Chain.FIRST),
+        loader(PeerGroup, ObjMapping, Chain.FIRST),
+        name_mapping(PeerGroup, map={
+            "local_as": "asnum",
+        }),
+    ]
+)
+
+to_bgp_global_options = retort.get_loader(GlobalOptions)
+to_interface_changes = retort.get_loader(InterfaceChanges)
+
+
+def to_bgp_peer(local: PeerDTO, connected: PeerDTO, connected_device: Device, interface: Optional[str]) -> Peer:
+    options = retort.load(local, PeerOptions)
+    # TODO validate `lagg_links` before conversion
+    result = Peer(
+        addr=str(ip_interface(connected.addr).ip),
+        interface=interface,
+        remote_as=ASN(connected.asnum),
+        hostname=connected_device.hostname,
+        options=options,
+    )
+    # connected
+    result.vrf_name = getattr(connected, "vrf", result.vrf_name)
+    result.group_name = getattr(connected, "group_name", result.group_name)
+    result.description = getattr(connected, "description", result.description)
+    result.families = getattr(connected, "families", result.families)
+    # local
+    result.import_policy = getattr(connected, "import_policy", result.import_policy)
+    result.export_policy = getattr(connected, "export_policy", result.export_policy)
+    result.update_source = getattr(connected, "update_source", result.update_source)
+    return result

annet/mesh/peer_models.py

@@ -0,0 +1,98 @@
+from typing import Literal, Annotated, Union, Optional
+
+from .basemodel import BaseMeshModel, Concat
+from ..bgp_models import BFDTimers
+
+FamilyName = Literal["ipv4_unicast", "ipv6_unicast", "ipv4_labeled", "ipv6_labeled"]
+
+
+class _SharedOptionsDTO(BaseMeshModel):
+    """
+    Options which can be set on connected pair or group of peers
+    """
+    add_path: bool
+    multipath: bool
+    advertise_irb: bool
+    send_labeled: bool
+    send_community: bool
+    bfd: bool
+    bfd_timers: BFDTimers
+
+
+class MeshSession(_SharedOptionsDTO):
+    """
+    Options which are set on connected pair
+    """
+    asnum: Union[int, str]
+    vrf: str
+    families: Annotated[set[FamilyName], Concat()]
+    group_name: str
+
+    bmp_monitor: bool
+
+    import_policy: str
+    export_policy: str
+
+
+class _OptionsDTO(_SharedOptionsDTO):
+    """
+    Options which can be set on group of peers or peer itself
+    """
+    unnumbered: bool
+    rr_client: bool
+    next_hop_self: bool
+    extended_next_hop: bool
+    send_lcommunity: bool
+    send_extcommunity: bool
+    import_limit: bool
+    teardown_timeout: bool
+    redistribute: bool
+    passive: bool
+    mtu_discovery: bool
+    advertise_inactive: bool
+    advertise_bgp_static: bool
+    allowas_in: bool
+    auth_key: bool
+    multihop: bool
+    multihop_no_nexthop_change: bool
+    af_no_install: bool
+    rib: bool
+    resolve_vpn: bool
+    af_rib_group: Optional[str]
+    af_loops: int
+    hold_time: int
+    listen_network: bool
+    remove_private: bool
+    as_override: bool
+    aigp: bool
+    no_prepend: bool
+    no_explicit_null: bool
+    uniq_iface: bool
+    advertise_peer_as: bool
+    connect_retry: bool
+    advertise_external: bool
+    listen_only: bool
+    soft_reconfiguration_inbound: bool
+    not_active: bool
+    mtu: int
+
+
+class PeerDTO(MeshSession, _OptionsDTO):
+    pod: int
+    addr: str
+    description: str
+
+    subif: int
+    lag: Optional[int]
+    lag_links_min: Optional[int]
+    svi: Optional[int]
+
+    group_name: str
+
+
+class MeshPeerGroup(_OptionsDTO):
+    name: str
+    remote_as: Union[int, str]
+    internal_name: str
+    update_source: str
+    description: str