annet 0.0__py3-none-any.whl

annet/generators/__init__.py

@@ -0,0 +1,435 @@
+from __future__ import annotations
+
+import dataclasses
+import importlib
+import os
+import textwrap
+from collections import OrderedDict as odict
+from typing import (
+    FrozenSet,
+    Iterable,
+    List,
+    Optional,
+    Union,
+)
+
+from annet.annlib.rbparser.acl import compile_acl_text
+from contextlog import get_logger
+
+from annet.storage import Device
+
+from annet import patching, tabparser, tracing
+from annet.cli_args import GenSelectOptions, ShowGeneratorsOptions
+from annet.lib import (
+    get_context,
+)
+from annet.tracing import tracing_connector
+from annet.types import (
+    GeneratorEntireResult,
+    GeneratorJSONFragmentResult,
+    GeneratorPartialResult,
+    GeneratorPartialRunArgs,
+    GeneratorResult,
+)
+from .base import (
+    BaseGenerator,
+    TextGenerator as TextGenerator,
+    ParamsList as ParamsList,
+)
+from .exceptions import NotSupportedDevice, GeneratorError
+from .jsonfragment import JSONFragment
+from .partial import PartialGenerator
+from .entire import Entire
+from .ref import RefGenerator
+from .perf import GeneratorPerfMesurer
+from .result import RunGeneratorResult
+
+# =====
+DISABLED_TAG = "disable"
+
+
+# =====
+def get_list(args: ShowGeneratorsOptions):
+    if args.generators_context is not None:
+        os.environ["ANN_GENERATORS_CONTEXT"] = args.generators_context
+    return {
+        cls.__class__.__name__: {
+            "type": cls.TYPE,
+            "tags": set(cls.TAGS),
+            "description": get_description(cls.__class__),
+        }
+        for cls in _get_generators(get_context()["generators"], None)
+    }
+
+
+def get_description(gen_cls) -> str:
+    return textwrap.dedent(" ".join([
+        (gen_cls.__doc__ or ""),
+        ("Disabled. Use '-g %s' to enable" % gen_cls.__name__ if DISABLED_TAG in gen_cls.TAGS else "")
+    ])).strip()
+
+
+def validate_genselect(gens: GenSelectOptions, all_classes):
+    logger = get_logger()
+    unknown_err = "Unknown generator alias %s"
+    all_aliases = {
+        alias
+        for cls in all_classes
+        for alias in cls.get_aliases()
+    }
+    for gen_set in (gens.allowed_gens, gens.force_enabled):
+        for alias in set(gen_set or ()) - all_aliases:
+            logger.error(unknown_err, alias)
+            raise Exception(unknown_err % alias)
+
+
+@dataclasses.dataclass
+class Generators:
+    """Collection of various types of generators."""
+
+    partial: List[PartialGenerator] = dataclasses.field(default_factory=list)
+    entire: List[Entire] = dataclasses.field(default_factory=list)
+    ref: List[RefGenerator] = dataclasses.field(default_factory=list)
+    json_fragment: List[JSONFragment] = dataclasses.field(default_factory=list)
+
+
+def build_generators(storage, gens: GenSelectOptions, device: Optional[Device] = None) -> Generators:
+    """Return generators that meet the gens filter conditions."""
+    if gens.generators_context is not None:
+        os.environ["ANN_GENERATORS_CONTEXT"] = gens.generators_context
+    all_generators = _get_generators(get_context()["generators"], storage, device)
+    ref_generators = _get_ref_generators(get_context()["generators"], storage, device)
+    validate_genselect(gens, all_generators)
+    classes = list(select_generators(gens, all_generators))
+    partial = [obj for obj in classes if obj.TYPE == "PARTIAL"]
+    entire = [obj for obj in classes if obj.TYPE == "ENTIRE"]
+    entire = list(sorted(entire, key=lambda x: x.prio, reverse=True))
+    json_fragment = [obj for obj in classes if obj.TYPE == "JSON_FRAGMENT"]
+    return Generators(
+        partial=partial,
+        entire=entire,
+        json_fragment=json_fragment,
+        ref=ref_generators,
+    )
+
+
+@tracing.function
+def run_partial_initial(device):
+    from .common.initial import InitialConfig
+
+    tracing_connector.get().set_device_attributes(tracing_connector.get().get_current_span(), device)
+
+    run_args = GeneratorPartialRunArgs(device)
+    return run_partial_generators([InitialConfig(storage=device.storage)], [], run_args)
+
+
+@tracing.function
+def run_partial_generators(
+        gens: List["PartialGenerator"],
+        ref_gens: List["RefGenerator"],
+        run_args: GeneratorPartialRunArgs,
+):
+    logger = get_logger(host=run_args.device.hostname)
+    tracing_connector.get().set_device_attributes(tracing_connector.get().get_current_span(), run_args.device)
+
+    ret = RunGeneratorResult()
+    if run_args.generators_context is not None:
+        os.environ["ANN_GENERATORS_CONTEXT"] = run_args.generators_context
+
+    for gen in ref_gens:
+        ret.ref_matcher.add(gen.ref(run_args.device), gen)
+
+    logger.debug("Generating selected PARTIALs ...")
+
+    for gen in gens:
+        try:
+            result = _run_partial_generator(gen, run_args)
+        except NotSupportedDevice as exc:
+            logger.info("generator %s raised unsupported error: %r", gen, exc)
+            continue
+
+        if not result:
+            continue
+
+        config = result.safe_config if run_args.use_acl_safe else result.config
+
+        ref_match = ret.ref_matcher.match(config)
+        for ref_gen, groups in ref_match:
+            gens.append(ref_gen.with_groups(groups))
+            ret.ref_track.add(gen.__class__, ref_gen.__class__)
+
+        ret.ref_track.config(gen.__class__, config)
+        ret.add_partial(result)
+
+    return ret
+
+
+@tracing.function(name="run_partial_generator")
+def _run_partial_generator(gen: "PartialGenerator", run_args: GeneratorPartialRunArgs) -> Optional[GeneratorPartialResult]:
+    logger = get_logger(generator=_make_generator_ctx(gen))
+    device = run_args.device
+    output = ""
+    config = odict()
+    safe_config = odict()
+
+    if not gen.supports_device(device):
+        logger.info("generator %s is not supported for device %s", gen, device.hostname)
+        return None
+
+    span = tracing_connector.get().get_current_span()
+    if span:
+        tracing_connector.get().set_device_attributes(span, run_args.device)
+        tracing_connector.get().set_dimensions_attributes(span, gen, run_args.device)
+        span.set_attributes({
+            "use_acl": run_args.use_acl,
+            "use_acl_safe": run_args.use_acl_safe,
+            "generators_context": str(run_args.generators_context),
+        })
+
+    with GeneratorPerfMesurer(gen, run_args=run_args) as pm:
+        if not run_args.no_new:
+            if gen.get_user_runner(device):
+                logger.info("Generating PARTIAL ...")
+            try:
+                output = gen(device, run_args.annotate)
+            except NotSupportedDevice:
+                # это исключение нужно передать выше в оригинальном виде
+                raise
+            except Exception as err:
+                filename, lineno = gen.get_running_line()
+                logger.exception("Generator error in file '%s:%i'", filename, lineno)
+                raise GeneratorError(f"{gen} on {device}") from err
+
+            fmtr = tabparser.make_formatter(device.hw)
+            try:
+                config = tabparser.parse_to_tree(text=output, splitter=fmtr.split)
+            except tabparser.ParserError as err:
+                logger.exception("Parser error")
+                raise GeneratorError from err
+
+    acl = gen.acl(device) or ""
+    rules = compile_acl_text(textwrap.dedent(acl), device.hw.vendor)
+    acl_safe = gen.acl_safe(device) or ""
+    safe_rules = compile_acl_text(textwrap.dedent(acl_safe), device.hw.vendor)
+
+    if run_args.use_acl:
+        try:
+            with tracing_connector.get().start_as_current_span("apply_acl", tracer_name=__name__, min_duration="0.01") as acl_span:
+                tracing_connector.get().set_device_attributes(acl_span, run_args.device)
+                config = patching.apply_acl(
+                    config=config,
+                    rules=rules,
+                    fatal_acl=True,
+                    with_annotations=run_args.annotate,
+                )
+            if run_args.use_acl_safe:
+                with tracing_connector.get().start_as_current_span(
+                    "apply_acl_safe",
+                    tracer_name=__name__,
+                    min_duration="0.01"
+                ) as acl_safe_span:
+                    tracing_connector.get().set_device_attributes(acl_safe_span, run_args.device)
+                    safe_config = patching.apply_acl(
+                        config=config,
+                        rules=safe_rules,
+                        fatal_acl=False,
+                        with_annotations=run_args.annotate,
+                    )
+        except patching.AclError as err:
+            logger.error("ACL error: generator is not allowed to yield this command: %s", err)
+            raise GeneratorError from err
+        except NotImplementedError as err:
+            logger.error(str(err))
+            raise GeneratorError from err
+
+    return GeneratorPartialResult(
+        name=gen.__class__.__name__,
+        tags=gen.TAGS,
+        output=output,
+        acl=acl,
+        acl_rules=rules,
+        acl_safe=acl_safe,
+        acl_safe_rules=safe_rules,
+        config=config,
+        safe_config=safe_config,
+        perf=pm.last_result,
+    )
+
+
+@tracing.function
+def check_entire_generators_required_packages(gens, device_packages: FrozenSet[str]) -> List[str]:
+    errors: List[str] = []
+    for gen in gens:
+        if not gen.REQUIRED_PACKAGES.issubset(device_packages):
+            missing = gen.REQUIRED_PACKAGES - device_packages
+            missing_str = ", ".join("`{}'".format(pkg) for pkg in sorted(missing))
+            if len(missing) == 1:
+                errors.append("missing package {} required for {}".format(missing_str, gen))
+            else:
+                errors.append("missing packages {} required for {}".format(missing_str, gen))
+    return errors
+
+
+@tracing.function
+def run_file_generators(
+        gens: Iterable[Union["JSONFragment", "Entire"]],
+        device: "Device",
+) -> RunGeneratorResult:
+    """Run generators that generate files or file parts."""
+    ret = RunGeneratorResult()
+    logger = get_logger(host=device.hostname)
+    logger.debug("Generating selected ENTIREs and JSON_FRAGMENTs ...")
+    for gen in gens:
+        if gen.__class__.TYPE == "ENTIRE":
+            run_generator_fn = _run_entire_generator
+            add_result_fn = ret.add_entire
+        elif gen.__class__.TYPE == "JSON_FRAGMENT":
+            run_generator_fn = _run_json_fragment_generator
+            add_result_fn = ret.add_json_fragment
+        else:
+            raise RuntimeError(f"Unknown generator class type: cls={gen.__class__} TYPE={gen.__class__.TYPE}")
+        try:
+            result = run_generator_fn(gen, device)
+        except NotSupportedDevice as exc:
+            logger.info("generator %s raised unsupported error: %r", gen, exc)
+            continue
+        if result:
+            add_result_fn(result)
+
+    return ret
+
+
+@tracing.function(min_duration="0.5")
+def _run_entire_generator(gen: "Entire", device: "Device") -> Optional[GeneratorResult]:
+    logger = get_logger(generator=_make_generator_ctx(gen))
+    if not gen.supports_device(device):
+        logger.info("generator %s is not supported for device %s", gen, device.hostname)
+        return
+
+    span = tracing_connector.get().get_current_span()
+    if span:
+        tracing_connector.get().set_device_attributes(span, device)
+        tracing_connector.get().set_dimensions_attributes(span, gen, device)
+
+    path = gen.path(device)
+    if not path:
+        raise RuntimeError("entire generator should return non-empty path")
+
+    logger.info("Generating ENTIRE ...")
+    with GeneratorPerfMesurer(gen, trace_min_duration="0.5") as pm:
+        output = gen(device)
+
+    return GeneratorEntireResult(
+        name=gen.__class__.__name__,
+        tags=gen.TAGS,
+        path=path,
+        output=output,
+        reload=gen.get_reload_cmds(device),
+        prio=gen.prio,
+        perf=pm.last_result,
+        is_safe=gen.is_safe(device),
+    )
+
+
+def _make_generator_ctx(gen):
+    return "%s.[%s]" % (gen.__module__, gen.__class__.__name__)
+
+
+def _run_json_fragment_generator(
+        gen: "JSONFragment",
+        device: "Device",
+) -> Optional[GeneratorResult]:
+    logger = get_logger(generator=_make_generator_ctx(gen))
+    if not gen.supports_device(device):
+        logger.info("generator %s is not supported for device %s", gen, device.hostname)
+        return
+
+    path = gen.path(device)
+    if not path:
+        raise RuntimeError("json fragment generator should return non-empty path")
+
+    acl_item_or_list_of_items = gen.acl(device)
+    safe_acl_item_or_list_of_items = gen.acl_safe(device)
+    if not acl_item_or_list_of_items:
+        raise RuntimeError("json fragment generator should return non-empty acl")
+    if isinstance(acl_item_or_list_of_items, list):
+        acl = acl_item_or_list_of_items
+    else:
+        acl = [acl_item_or_list_of_items]
+    if isinstance(safe_acl_item_or_list_of_items, list):
+        acl_safe = safe_acl_item_or_list_of_items
+    else:
+        acl_safe = [safe_acl_item_or_list_of_items]
+
+    logger.info("Generating JSON_FRAGMENT ...")
+    with GeneratorPerfMesurer(gen) as pm:
+        config = gen(device)
+    reload_cmds = gen.get_reload_cmds(device)
+    return GeneratorJSONFragmentResult(
+        name=gen.__class__.__name__,
+        tags=gen.TAGS,
+        path=path,
+        acl=acl,
+        acl_safe=acl_safe,
+        config=config,
+        reload=reload_cmds,
+        perf=pm.last_result,
+        reload_prio=gen.reload_prio,
+    )
+
+
+def _get_generators(module_paths: Union[List[str], dict], storage, device=None):
+    if isinstance(module_paths, dict):
+        if device is None:
+            module_paths = module_paths.get("default")
+        else:
+            modules = []
+            seen = set()
+            for prop, prop_modules in module_paths.get("per_device_property", {}).items():
+                if getattr(device, prop, False) is True:
+                    for module in prop_modules:
+                        if module not in seen:
+                            modules.append(module)
+                            seen.add(module)
+            module_paths = modules or module_paths.get("default")
+    res_generators = []
+    for module_path in module_paths:
+        module = importlib.import_module(module_path)
+        if hasattr(module, "get_generators"):
+            generators: List[BaseGenerator] = module.get_generators(storage)
+            res_generators += generators
+    return res_generators
+
+
+def _get_ref_generators(module_paths: List[str], storage, device):
+    if isinstance(module_paths, dict):
+        module_paths = module_paths.get("default")
+    res_generators = []
+    for module_path in module_paths:
+        module = importlib.import_module(module_path)
+        if hasattr(module, "get_ref_generators"):
+            res_generators += module.get_ref_generators(storage)
+    return res_generators
+
+
+def select_generators(gens: GenSelectOptions, classes: Iterable[BaseGenerator]):
+    def contains(obj, where):
+        if where:
+            return obj.get_aliases().intersection(where)
+        return False
+
+    def has(cls, what):
+        return what in cls.TAGS
+
+    flts = [lambda c: not isinstance(c, RefGenerator)]
+    if gens.allowed_gens:
+        flts.append(lambda c: contains(c, gens.allowed_gens))
+    elif gens.force_enabled:
+        flts.append(lambda c: not has(c, DISABLED_TAG) or contains(c, gens.force_enabled))
+    elif not gens.ignore_disabled:
+        flts.append(lambda c: not has(c, DISABLED_TAG))
+
+    if gens.excluded_gens:
+        flts.append(lambda c: not contains(c, gens.excluded_gens))
+
+    return filter(lambda x: all(f(x) for f in flts), classes)

annet/generators/base.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import abc
+import contextlib
+import textwrap
+from typing import Union, List
+
+from annet import tabparser, tracing
+from annet.tracing import tracing_connector
+from .exceptions import InvalidValueFromGenerator
+
+
+class DefaultBlockIfCondition:
+    pass
+
+
+ParamsList = tabparser.JuniperList
+
+
+class GenStringable(abc.ABC):
+    @abc.abstractmethod
+    def gen_str(self) -> str:
+        pass
+
+
+def _filter_str(value: Union[
+    str, int, float, tabparser.JuniperList, ParamsList, GenStringable]):
+    if isinstance(value, (
+            str,
+            int,
+            float,
+            tabparser.JuniperList,
+            ParamsList,
+    )):
+        return str(value)
+
+    if hasattr(value, "gen_str") and callable(value.gen_str):
+        return value.gen_str()
+
+    raise InvalidValueFromGenerator(
+        "Invalid yield type: %s(%s)" % (type(value).__name__, value))
+
+
+def _split_and_strip(text):
+    if "\n" in text:
+        rows = textwrap.dedent(text).strip().split("\n")
+    else:
+        rows = [text]
+    return rows
+
+
+# =====
+class BaseGenerator:
+    TYPE: str
+    TAGS: List[str]
+
+    def supports_device(self, device) -> bool:  # pylint: disable=unused-argument
+        return True
+
+
+class TreeGenerator(BaseGenerator):
+    def __init__(self, indent="  "):
+        self._indents = []
+        self._rows = []
+        self._block_path = []
+        self._indent = indent
+
+    @tracing.contextmanager(min_duration="0.1")
+    @contextlib.contextmanager
+    def block(self, *tokens, indent=None):
+        span = tracing_connector.get().get_current_span()
+        if span:
+            span.set_attribute("tokens", " ".join(map(str, tokens)))
+
+        indent = self._indent if indent is None else indent
+        block = " ".join(map(_filter_str, tokens))
+        self._block_path.append(block)
+        self._append_text(block)
+        self._indents.append(indent)
+        yield
+        self._indents.pop(-1)
+        self._block_path.pop(-1)
+
+    @contextlib.contextmanager
+    def block_if(self, *tokens, condition=DefaultBlockIfCondition):
+        if condition is DefaultBlockIfCondition:
+            condition = (None not in tokens and "" not in tokens)
+        if condition:
+            with self.block(*tokens):
+                yield
+                return
+        yield
+
+    @contextlib.contextmanager
+    def multiblock(self, *blocks):
+        if blocks:
+            blk = blocks[0]
+            tokens = blk if isinstance(blk, (list, tuple)) else [blk]
+            with self.block(*tokens):
+                with self.multiblock(*blocks[1:]):
+                    yield
+                    return
+        yield
+
+    @contextlib.contextmanager
+    def multiblock_if(self, *blocks, condition=DefaultBlockIfCondition):
+        if condition is DefaultBlockIfCondition:
+            condition = (None not in blocks)
+            if condition:
+                if blocks:
+                    blk = blocks[0]
+                    tokens = blk if isinstance(blk, (list, tuple)) else [blk]
+                    with self.block(*tokens):
+                        with self.multiblock(*blocks[1:]):
+                            yield
+                            return
+        yield
+
+    # ===
+    def _append_text(self, text):
+        self._append_text_cb(text)
+
+    def _append_text_cb(self, text, row_cb=None):
+        for row in _split_and_strip(text):
+            if row_cb:
+                row = row_cb(row)
+            self._rows.append("".join(self._indents) + row)
+
+
+class TextGenerator(TreeGenerator):
+    def __add__(self, line):
+        self._append_text(line)
+        return self
+
+    def __iter__(self):
+        yield from self._rows

annet/generators/common/initial.py

@@ -0,0 +1,33 @@
+from annet.generators import PartialGenerator
+
+
+class InitialConfig(PartialGenerator):
+
+    """
+    Конфиги у свежих (еще ни разу не настраиваемых устройств)
+    на самом деле НЕ пустые. В данном генераторе отображен
+    такой набор команд, по крайней мере тех, которые могут
+    изменяться в ходе первичной конфигурации.
+
+    Acl для данного генератора не нужен, он будет генерировать
+    конфиг целиком.
+    """
+    def __init__(self, storage=None):
+        self._do_run = not storage
+        super().__init__(storage=storage)
+
+    def run_huawei(self, device):
+        if not self._do_run:
+            return
+        if device.hw.CE:
+            yield """
+            telnet server disable
+            telnet ipv6 server disable
+            diffserv domain default
+            aaa
+                authentication-scheme default
+                authorization-scheme default
+                accounting-scheme default
+                domain default
+                domain default_admin
+            """

annet/generators/entire.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import pkgutil
+import re
+import types
+from typing import (
+    FrozenSet,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Union,
+)
+
+from annet.lib import (
+    flatten,
+    jinja_render,
+    mako_render,
+)
+from .base import BaseGenerator, _filter_str
+from .exceptions import NotSupportedDevice
+
+
+class Entire(BaseGenerator):
+    TYPE = "ENTIRE"
+    TAGS: List[str] = []
+    REQUIRED_PACKAGES: FrozenSet[str] = frozenset()
+
+    def __init__(self, storage):
+        self.storage = storage
+        # между генераторами для одного и того же path - выбирается тот что больше
+        if not hasattr(self, "prio"):
+            self.prio = 100
+        self.__device = None
+
+    def supports_device(self, device):
+        return bool(self.path(device))
+
+    def run(self, device) -> Union[None, str, Iterable[Union[str, tuple]]]:
+        raise NotImplementedError
+
+    def reload(self, device) -> Optional[
+        str]:  # pylint: disable=unused-argument
+        return
+
+    def get_reload_cmds(self, device) -> str:
+        ret = self.reload(device) or ""
+        path = self.path(device)
+        if path and device.hw.PC and device.hw.soft.startswith(
+            ("Cumulus", "SwitchDev", "SONiC"),
+        ):
+            parts = []
+            if ret:
+                parts.append(ret)
+            parts.append("/usr/bin/etckeeper commitreload %s" % path)
+            return "\n".join(parts)
+        return ret
+
+    def path(self, device) -> Optional[str]:
+        raise NotImplementedError("Required PATH for ENTIRE generator")
+
+    # pylint: disable=unused-argument
+    def is_safe(self, device) -> bool:
+        """Output gen results when --acl-safe flag is used"""
+        return False
+
+    def read(self, path) -> str:
+        return pkgutil.get_data(__name__, path).decode()
+
+    def mako(self, text, **kwargs) -> str:
+        return mako_render(text, dedent=True, device=self.__device, **kwargs)
+
+    def jinja(self, text, **kwargs) -> str:
+        return jinja_render(text, dedent=True, device=self.__device, **kwargs)
+
+    # =====
+
+    @classmethod
+    def get_aliases(cls) -> Set[str]:
+        return {cls.__name__, *cls.TAGS}
+
+    def __call__(self, device):
+        self.__device = device
+        parts = []
+        run_res = self.run(device)
+        if isinstance(run_res, str):
+            run_res = (run_res,)
+        if run_res is None or not isinstance(run_res, (tuple, types.GeneratorType)):
+            raise Exception("generator %s returns %s" % (
+                self.__class__.__name__, type(run_res)))
+        for text in run_res:
+            if isinstance(text, tuple):
+                text = " ".join(map(_filter_str, flatten(text)))
+            assert re.search(r"\bNone\b", text) is None, \
+                "Found 'None' in yield result: %s" % text
+            parts.append(text)
+        return "\n".join(parts)