annet 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

annet/executor.py

@@ -1,29 +1,13 @@
 import asyncio
-import logging
-import multiprocessing
 import os
-import platform
-import resource
-import signal
 import statistics
-import time
 from abc import ABC, abstractmethod
 from functools import partial
 from operator import itemgetter
-from queue import Empty
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
 
 import colorama
-import psutil
-
-import annet.lib
 from annet.annlib.command import Command, CommandList, Question  # noqa: F401
-from annet.storage import Device
-
-
-_logger = logging.getLogger(__name__)
-FIRST_EXCEPTION = 1
-ALL_COMPLETED = 2
 
 
 class CommandResult(ABC):
@@ -54,19 +38,6 @@ class Connector(ABC):
         pass
 
 
-class Executor(ABC):
-    # method for bulk config downloading TODO: remove in favor Connector.cmd
-    @abstractmethod
-    def fetch(self,
-              devices: List[Device],
-              files_to_download: Dict[str, List[str]] = None) -> Tuple[Dict[Device, str], Dict[Device, Any]]:
-        pass
-
-    @abstractmethod
-    async def amake_connection(self, device: Device) -> Connector:
-        pass
-
-
 class ExecutorException(Exception):
     def __init__(self, *args: List[Any], auxiliary: Optional[Any] = None, **kwargs: object):
         self.auxiliary = auxiliary
@@ -103,117 +74,6 @@ class NonzeroRetcode(ExecException):
 class CommitException(ExecException):
     pass
 
-def chunks_tuple(l, n):  # noqa
-    return [tuple(l[i:i + n]) for i in range(0, len(l), n)]
-
-
-def async_bulk(
-        executor: Executor,
-        devices: List[Device],
-        coro_gen: Callable[[Connector, Device], Any],
-        *args,
-        processes: int = 1,
-        show_report: bool = True,
-        do_log: bool = True,
-        log_dir: Optional[str] = None,
-        policy: int = ALL_COMPLETED,
-        **kwargs,
-):
-    """Connect to specified devices and work with their CLI.
-
-    Note: this function is not allowed to be run in parallel, since it's using global state (TODO: fixme)
-
-    :param devices: List of devices' fqdns to use their CLI.
-    :param coro_gen: Async function. It contains all logic about usage of CLI.
-        See docstring of "bind_coro_args" for allowed function signature and examples.
-    :param args: Positional arguments to "bulk" function.
-    :type processes: Amount of processes to fork for current work.
-    :param show_report: Set this flag to show report to stdout.
-    :param do_log: If True and log_dir is not set, then log_dir will be filled automatically.
-    :param log_dir: Specify path to log all response from devices.
-    :param policy: int flag. If FIRST_EXCEPTION, then work will be stopped after first error.
-        Otherwise all hosts will be processed.
-        TODO: fix that policy is not used if processes=1
-    :param kwargs: other arguments to pass to "bulk" function.
-        Note: it is not passed directly to "coro_gen" function!
-            kwargs should be {'kwargs': {'var1': value1}} to set "var1" with "value1" in "coro_gen" function.
-
-    TODOs:
-        * do not log if do_log=False and log_dir is set.
-
-    """
-    res = {}
-    deploy_durations = {}
-    kwargs["log_dir"] = log_dir
-    kwargs["policy"] = policy
-
-    if processes == 1:
-        host_res, host_duration = annet.lib.do_async(bulk(executor, devices, coro_gen, *args, **kwargs))
-        res.update(host_res)
-        deploy_durations.update(host_duration)
-    else:
-        # FIXME: show_report works per process
-        if len(devices) != len(set(devices)):
-            raise Exception("hostnames should be unique")
-        # warm up a cache
-        # asyncio.get_event_loop().run_until_complete(get_validator_rt_data(hostnames))
-        if isinstance(devices, dict):
-            devices = list(devices.keys())
-        hostnames_chunks = chunks_tuple(devices, int(len(devices) / processes) + 1)
-        pool = {}
-        for hostnames_chunk in hostnames_chunks:
-            res_q = multiprocessing.Queue()
-            p = multiprocessing.Process(target=_mp_async_bulk, args=[res_q, hostnames_chunk, coro_gen, *args], kwargs=kwargs)
-            pool[p] = [res_q, hostnames_chunk]
-            p.start()
-            _logger.info("process (id=%d) work with %d chunks", p.pid, len(hostnames_chunks))
-
-        seen_error = False
-        while True:
-            done = []
-            for p in pool:
-                host_res = None
-                try:
-                    # proc wont be exited till q.get() call
-                    host_res, host_duration = pool[p][0].get(timeout=0.2)
-                except Empty:
-                    pass
-                else:
-                    done.append(p)
-
-                if not p.is_alive() and not host_res:
-                    _logger.error("process %s has died: hostnames: %s", p.pid, pool[p][1])
-                    host_res = {hostname: Exception("died with exitcode %s" % p.exitcode) for hostname in pool[p][1]}
-                    host_duration = {hostname: 0 for hostname in pool[p][1]}  # FIXME:
-                    done.append(p)
-
-                if host_res:
-                    res.update(host_res)
-                    deploy_durations.update(host_duration)
-
-                if p.exitcode:
-                    _logger.error("process %s finished with bad exitcode %s", p.pid, p.exitcode)
-                    seen_error = True
-            for p in done:
-                pool.pop(p)
-            if policy == FIRST_EXCEPTION and seen_error:
-                for p in pool:
-                    p.terminate()
-                    if p.is_alive():
-                        time.sleep(0.4)
-                    if p.is_alive():
-                        os.kill(p.pid, signal.SIGKILL)
-                    for hostname in pool[p][1]:
-                        res[hostname] = Exception("force kill with exitcode %s" % p.exitcode)
-                        deploy_durations[hostname] = 0  # FIXME:
-            if not pool:
-                break
-
-    if show_report:
-        show_bulk_report(devices, res, deploy_durations, do_log and log_dir)
-
-    return res
-
 
 def _show_type_summary(caption, items, total, stat_items=None):
     if items:
@@ -314,139 +174,6 @@ def _print_failed(host, res):
     print("  %s - %s" % (color + host + colorama.Style.RESET_ALL, _format_exc(exc)))
 
 
-def _mp_async_bulk(res_q: multiprocessing.Queue, *args, **kwargs):
-    res = annet.lib.do_async(bulk(*args, **kwargs))
-    res_q.put(res)
-    res_q.close()
-
-
-async def bulk(
-        executor: Executor,
-        devices: List[Device],
-        coro_gen: Callable[[Connector, Device, Optional[Dict[str, Any]]], Any],
-        max_parallel: float = 100,
-        policy: int = ALL_COMPLETED,
-        log_dir: str = True,  # pylint: disable=unused-argument
-        kwargs: Optional[dict] = None,
-        console_log: bool = True
-) -> Tuple[Dict[str, Any], Dict[str, float]]:
-    """Connect to specified devices and work with their CLI.
-
-    :param hostnames: List of devices' fqdns to use their CLI.
-    :param coro_gen: Async function. It contains all logic about usage of CLI.
-        See docstring of "bind_coro_args" for allowed function signature and examples.
-    :param max_parallel: Upper border to CPU usage (in percentage 1 CPU = 100).
-        If cpu usage is over, then tasks are trottled.
-    :param policy: Flag to specify when tasks are completed.
-    :param log_dir: Specify path to log all response from devices.
-        TODO: fix default value.
-    :param kwargs: Device independent arguments to call function. See @bind_coro_args for details.
-    :param get_device: See "make_connection" for better understanding.
-    :param device_cls: See "make_connection" for better understanding.
-    :param streamer_cls: See "make_connection" for better understanding.
-    :param console_log: If True and there is no handlers for root logger, then stderr will be used for logging.
-    :return: two dicts with results per host and execution duration per host.
-
-    """
-    if console_log:
-        init_log()
-
-    tasks = []
-    res = {}
-    pending = set()
-    tasks_to_device = {}
-    time_of_start = {}
-    deploy_durations = {}
-    now = None
-    if not kwargs:
-        kwargs = {}
-
-    def start_hook(device: Device):
-        time_of_start[device.hostname] = time.monotonic()
-
-    def end_hook(device: Device, task: asyncio.Task):
-        duration = now - time_of_start[device.hostname]
-        deploy_durations[device.hostname] = duration
-
-        coro_exc = task.exception()
-        if coro_exc:
-            if policy == FIRST_EXCEPTION:
-                _logger.error("%s %r", device.hostname, coro_exc, exc_info=coro_exc)
-                _logger.info("Terminating all running tasks according to FIRST_EXCEPTION policy")
-                res[device.hostname] = coro_exc
-                raise CancelAllTasks
-            else:
-                if isinstance(coro_exc, AssertionError):
-                    _logger.error("%s %r", device.hostname, coro_exc, exc_info=coro_exc)
-                else:
-                    _logger.error("%s %r", device.hostname, coro_exc)
-            return coro_exc
-        else:
-            _logger.info("Finished in %0.2fs, hostname=%s", duration, device.hostname)
-            return task.result()
-
-    for device in devices:
-        try:
-            conn = await executor.amake_connection(device=device)
-        except Exception as exc:
-            _logger.error("failed to connect to %s %r", device.hostname, exc)
-            res[device] = exc
-            continue
-        start_hook(device)
-        task = asyncio.create_task(coro_gen(conn=conn, device=device, **kwargs))
-        tasks_to_device[task] = device
-        tasks.append(task)
-    try:
-        ndone = 0
-        with CpuThrottler(asyncio.get_event_loop(), maximum=max_parallel) as throttler:
-            while pending or tasks:
-                left_fds = int(fd_left() / 2)  # better to be safe than sorry
-
-                for _ in range(min(throttler.curr - len(pending), len(tasks), left_fds)):
-                    pending.add(tasks.pop(0))
-                if len(pending) == 0:
-                    _logger.debug("empty pending list. tasks=%s throttler curr=%s left_fds=%s. waiting", len(tasks),
-                                  throttler.curr, left_fds)
-                    await asyncio.sleep(1)
-                    continue
-                example_host = next(iter(pending))
-                _logger.debug("tasks status: %d pending, %d queued, pending example %s", len(pending), len(tasks), example_host)
-                done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
-
-                now = time.monotonic()
-                for task in done:
-                    device = tasks_to_device[task]
-                    res[device] = end_hook(device, task)
-                    ndone += 1
-    except CancelAllTasks:
-        exc = asyncio.CancelledError()
-
-        now = time.monotonic()
-        for device, task in _get_remaining(tasks, pending, tasks_to_device):
-            res[device] = exc
-
-            if device.hostname in time_of_start:
-                duration = now - time_of_start[device.hostname]
-            else:
-                duration = None
-            deploy_durations[device.hostname] = duration
-
-            if not asyncio.iscoroutine(task):
-                _logger.info("task %s", task)
-                task.cancel()
-
-    return res, deploy_durations
-
-
-def init_log():
-    streamer = logging.StreamHandler()
-    fmt = logging.Formatter("%(asctime)s - %(filename)s:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s",
-                            "%Y-%m-%d %H:%M:%S")
-    streamer.setFormatter(fmt)
-    if not logging.root.handlers:
-        logging.root.addHandler(streamer)
-
-
 class DeferredFileWrite:
     def __init__(self, file, mode="r"):
         self._file = file
@@ -465,87 +192,3 @@ class DeferredFileWrite:
 
     def flush(self):
         pass
-
-
-class CancelAllTasks(Exception):
-    pass
-
-
-def _get_remaining(tasks, pending, tasks_to_device):
-    for task in pending:
-        yield (tasks_to_device[task], task)
-    for task in tasks:
-        yield (tasks_to_device[task], task)
-
-
-_platform = platform.system()
-_fd_available = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
-
-
-def fd_left():
-    res = _fd_available
-    if _platform == "Linux":
-        res = _fd_available - len(list(os.scandir(path="/proc/self/fd/")))
-    return res
-
-
-class CpuThrottler:
-    def __init__(self, loop, start=20, maximum=None, minimum=5, hz=1.0, target=80.0):
-        self.loop = loop
-        self.minimum = int(minimum)
-        self.maximum = int(maximum or 0)
-        self.hz = hz
-        self.target = target
-        self.timer_handle = None
-        self.last_usage = 0
-        self.curr = int(start)
-        self.proc = psutil.Process(os.getpid())
-
-    def __enter__(self):
-        if self.proc and self.maximum:
-            self.proc.cpu_percent()  # initialize previous value
-            self.timer_handle = self.loop.call_later(self.hz, self.schedule)
-        return self
-
-    def __exit__(self, type_, value, tb):
-        if self.timer_handle:
-            self.timer_handle.cancel()
-
-    def schedule(self):
-        # re-schedule
-        self.timer_handle = self.loop.call_later(self.hz, self.schedule)
-
-        cpu_usage = self.proc.cpu_percent()
-        self.last_usage = cpu_usage
-        _logger.debug("current cpu_usage=%s", cpu_usage)
-        if cpu_usage > self.target:
-            self.change_by(0.5)
-        elif cpu_usage > self.target * 0.8:
-            pass
-        elif cpu_usage > self.target * 0.2:
-            self.change_by(1.2)
-        else:
-            self.change_by(1.5)
-
-    def change_by(self, rate):
-        new_curr = int(self.curr * rate)
-        # округлим в нужную сторону
-        if new_curr == self.curr:
-            if rate > 1:
-                new_curr += 1
-            elif rate < 1:
-                new_curr -= 1
-        # ограничим пределами
-        if new_curr < self.curr:
-            new_curr = max(self.minimum, new_curr)
-        else:
-            if self.maximum is not None:
-                new_curr = min(self.maximum, new_curr)
-
-        if new_curr < self.curr:
-            _logger.info("decreasing max_slots %d -> %d, cpu_usage=%.1f", self.curr, new_curr, self.last_usage)
-        elif new_curr > self.curr:
-            _logger.info("increasing max_slots %d -> %d, cpu_usage=%.1f", self.curr, new_curr, self.last_usage)
-
-        # new_curr не делаем меньше 0, иначе не сможем его увеличить
-        self.curr = max(new_curr, 1)

annet/mesh/device_models.py

@@ -76,10 +76,21 @@ class VrfOptions(_FamiliesMixin, BaseMeshModel):
     groups: Annotated[dict[str, MeshPeerGroup], DictMerge(Merge())]
 
 
+class L2VpnOptions(BaseMeshModel):
+    name: str
+    vid: str | int  # VLAN ID, possible values are 1 to 4094, ranges can be set as strings
+    l2vni: int  # VNI, possible values are 1 to 2**24-1
+    route_distinguisher: str  # like in VrfOptions
+    rt_import: Annotated[tuple[str, ...], Concat()]  # like in VrfOptions
+    rt_export: Annotated[tuple[str, ...], Concat()]  # like in VrfOptions
+    advertise_host_routes: bool  # advertise IP+MAC routes into L3VNI
+
+
 class GlobalOptionsDTO(_FamiliesMixin, BaseMeshModel):
     def __init__(self, **kwargs):
         kwargs.setdefault("groups", KeyDefaultDict(lambda x: MeshPeerGroup(name=x)))
         kwargs.setdefault("vrf", KeyDefaultDict(lambda x: VrfOptions(vrf_name=x)))
+        kwargs.setdefault("l2vpn", KeyDefaultDict(lambda x: L2VpnOptions(name=x)))
         super().__init__(**kwargs)
 
     as_path_relax: bool
@@ -89,3 +100,4 @@ class GlobalOptionsDTO(_FamiliesMixin, BaseMeshModel):
     router_id: str
     vrf: Annotated[dict[str, VrfOptions], DictMerge(Merge())]
     groups: Annotated[dict[str, MeshPeerGroup], DictMerge(Merge())]
+    l2vpn: Annotated[dict[str, L2VpnOptions], DictMerge(Merge())]

annet/mesh/models_converter.py

@@ -7,7 +7,7 @@ from adaptix import Retort, loader, Chain, name_mapping, as_is_loader
 from .peer_models import DirectPeerDTO, IndirectPeerDTO, VirtualPeerDTO, VirtualLocalDTO
 from ..bgp_models import (
     Aggregate, GlobalOptions, VrfOptions, FamilyOptions, Peer, PeerGroup, ASN, PeerOptions,
-    Redistribute, BFDTimers,
+    Redistribute, BFDTimers, L2VpnOptions, VidCollection,
 )
 
 
@@ -49,8 +49,10 @@ retort = Retort(
     recipe=[
         loader(InterfaceChanges, ObjMapping, Chain.FIRST),
         loader(ASN, ASN),
+        loader(VidCollection, VidCollection.parse),
         loader(GlobalOptions, ObjMapping, Chain.FIRST),
         loader(VrfOptions, ObjMapping, Chain.FIRST),
+        loader(L2VpnOptions, ObjMapping, Chain.FIRST),
         loader(FamilyOptions, ObjMapping, Chain.FIRST),
         loader(Aggregate, ObjMapping, Chain.FIRST),
         loader(PeerOptions, ObjMapping, Chain.FIRST),

annet/rulebook/arista/aaa.py

@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+
+from annet.annlib.types import Op
+
+
+@dataclass
+class UserConfig:
+    name: str
+    privilege: int
+    role: str
+    secret_type: str
+    secret: str
+
+
+def _parse_user_config(config_line):
+    """Convert a user config line into a dataclass. Config example:
+
+    username someuser privilege 15 role network-admin secret sha512 $6$....
+
+    privilege could be omitted if equal to 1
+    role could be omitted
+    secret could be omitted, 'nopassword' is provided instead
+    """
+    splstr = config_line.split()
+    name = splstr[1]
+    priv = 1
+    role = ""
+    secret_type = ""
+    secret = ""
+    if "privilege" in splstr:
+        pos = splstr.index("privilege")
+        priv = int(splstr[pos + 1])
+    if "role" in splstr:
+        pos = splstr.index("role")
+        role = splstr[pos + 1]
+    if "secret" in splstr:
+        pos = splstr.index("secret")
+        secret_type = splstr[pos + 1]
+        secret = splstr[pos + 2]
+    return UserConfig(name=name, privilege=priv, role=role, secret_type=secret_type, secret=secret)
+
+
+def user(key, diff, **_):
+    if diff[Op.ADDED] and not diff[Op.REMOVED]:
+        for add in diff[Op.ADDED]:
+            yield (True, add["row"], None)
+    elif diff[Op.REMOVED] and not diff[Op.ADDED]:
+        for _ in diff[Op.REMOVED]:
+            yield (False, f"no username {key[0]}", None)
+    else:
+        for num, add in enumerate(diff[Op.ADDED]):
+            new_user = _parse_user_config(add["row"])
+            old_user = _parse_user_config(diff[Op.REMOVED][num]["row"])
+            if new_user.privilege != old_user.privilege:
+                yield (True, f"username {key[0]} privilege {new_user.privilege}", None)
+            if new_user.role != old_user.role and not new_user.role:
+                yield (True, f"no username {key[0]} role", None)
+            elif new_user.role != old_user.role:
+                yield (True, f"username {key[0]} role {new_user.role}", None)
+            if new_user.secret != old_user.secret and not new_user.secret:
+                yield (True, f"username {key[0]} nopassword", None)
+            elif new_user.secret != old_user.secret:
+                yield (True, f"username {key[0]} secret {new_user.secret_type} {new_user.secret}", None)