annet 0.15.2__tar.gz → 0.15.3__tar.gz

{annet-0.15.2/annet.egg-info → annet-0.15.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: annet
-Version: 0.15.2
+Version: 0.15.3
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

{annet-0.15.2 → annet-0.15.3}/annet/adapters/netbox/common/storage_opts.py

@@ -1,6 +1,8 @@
 import os
 from typing import Any
 
+DEFAULT_URL = "http://localhost"
+
 
 class NetboxStorageOpts:
     def __init__(self, url: str, token: str):
@@ -9,7 +11,6 @@ class NetboxStorageOpts:
 
     @classmethod
     def parse_params(cls, conf_params: dict[str, str] | None, cli_opts: Any):
-        return cls(
-            url=os.getenv("NETBOX_URL", "http://localhost"),
-            token=os.getenv("NETBOX_TOKEN", "").strip(),
-        )
+        url = os.getenv("NETBOX_URL") or conf_params.get("url") or DEFAULT_URL
+        token = os.getenv("NETBOX_TOKEN", "").strip() or conf_params.get("token") or ""
+        return cls(url=url, token=token)

{annet-0.15.2 → annet-0.15.3}/annet/adapters/netbox/provider.py

@@ -1,6 +1,9 @@
+from typing import Dict, Any, Optional
+
 from dataclass_rest.exceptions import ClientError
 
 from annet.storage import StorageProvider, Storage
+from annet.connectors import AdapterWithName, AdapterWithConfig, T
 from .common.status_client import NetboxStatusClient
 from .common.storage_opts import NetboxStorageOpts
 from .common.query import NetboxQuery
@@ -23,7 +26,14 @@ def storage_factory(opts: NetboxStorageOpts) -> Storage:
         raise ValueError(f"Unsupported version: {status.netbox_version}")
 
 
-class NetboxProvider(StorageProvider):
+class NetboxProvider(StorageProvider, AdapterWithName, AdapterWithConfig):
+    def __init__(self, url: Optional[str] = None, token: Optional[str] = None):
+        self.url = url
+        self.token = token
+
+    def with_config(self, **kwargs: Dict[str, Any]) -> T:
+        return NetboxProvider(**kwargs)
+
     def storage(self):
         return storage_factory
 

{annet-0.15.2 → annet-0.15.3}/annet/annlib/tabparser.py

@@ -490,10 +490,11 @@ class RosFormatter(CommonFormatter):
                 rows.append((row, None, row_context))
 
         prev_prow = None
+        prev_prow_context = {}
         for sub_config, row_group in itertools.groupby(rows, lambda x: x[1]):
             if sub_config is None:
                 if prev_prow:
-                    yield prev_prow
+                    yield prev_prow, prev_prow_context
                     yield BlockBegin, None
                 for row, _, row_context in row_group:
                     yield row, row_context
@@ -502,7 +503,7 @@ class RosFormatter(CommonFormatter):
             else:
                 for row, _, row_context in row_group:
                     if context and context.parent and context.parent.row:
-                        prev_prow = context.parent.row
+                        prev_prow, prev_prow_context = context.parent.current
                         prow = f"{context.parent.row} {row}"
                     else:
                         prow = row

{annet-0.15.2 → annet-0.15.3}/annet/api/__init__.py

@@ -218,8 +218,8 @@ def log_host_progress_cb(pool: Parallel, task_result: TaskResult):
         stacklevel=2,
     )
     args = cast(cli_args.QueryOptions, pool.args[0])
-    connector, connector_opts = get_storage()
-    storage_opts = connector.opts().parse_params(connector_opts, args)
+    connector = get_storage()
+    storage_opts = connector.opts().parse_params({}, args)
     with connector.storage()(storage_opts) as storage:
         fqdns = storage.resolve_fdnds_by_query(args.query)
     PoolProgressLogger(device_fqdns=fqdns)(pool, task_result)
@@ -260,7 +260,7 @@ def patch(args: cli_args.ShowPatchOptions, loader: ann_gen.Loader):
     """ Сгенерировать патч для устройств """
     global live_configs  # pylint: disable=global-statement
     if args.config == "running":
-        fetcher = annet.deploy.fetcher_connector.get()
+        fetcher = annet.deploy.get_fetcher()
         live_configs = fetcher.fetch(loader.devices, processes=args.parallel)
     stdin = args.stdin(filter_acl=args.filter_acl, config=args.config)
 
@@ -434,7 +434,7 @@ class CliDeployerJob(DeployerJob):
         self.cmd_lines.extend(["= %s " % device.hostname, ""])
         self.cmd_lines.extend(map(itemgetter(-1), cmds))
         self.cmd_lines.append("")
-        deployer_driver = annet.deploy.driver_connector.get()
+        deployer_driver = annet.deploy.get_deployer()
         self.deploy_cmds[device] = deployer_driver.apply_deploy_rulebook(
             device.hw, cmds,
             do_commit=not self.args.dont_commit
@@ -494,7 +494,7 @@ class PCDeployerJob(DeployerJob):
                 "generator_types": generator_types,
             }
             self.diffs[device] = upload_files
-            deployer_driver = annet.deploy.driver_connector.get()
+            deployer_driver = annet.deploy.get_deployer()
             before, after = deployer_driver.build_configuration_cmdlist(device.hw)
             for cmd in deployer_driver.build_exit_cmdlist(device.hw):
                 after.add_cmd(cmd)

{annet-0.15.2 → annet-0.15.3}/annet/cli.py

@@ -15,7 +15,7 @@ import yaml
 from contextlog import get_logger
 from valkit.python import valid_logging_level
 
-from annet.deploy import driver_connector, fetcher_connector
+from annet.deploy import get_fetcher, get_deployer
 from annet import api, cli_args, filtering, generators
 from annet.api import collapse_texts, Deployer
 from annet.argparse import ArgParser, subcommand
@@ -68,8 +68,8 @@ def get_loader(gen_args: cli_args.GenOptions, args: cli_args.QueryOptionsBase):
     exit_stack = ExitStack()
     storages = []
     with exit_stack:
-        connector, connector_opts = get_storage()
-        storage_opts = connector.opts().parse_params(connector_opts, args)
+        connector, conf_params = get_storage()
+        storage_opts = connector.opts().parse_params(conf_params, args)
         storages.append(exit_stack.enter_context(connector.storage()(storage_opts)))
         yield Loader(*storages, args=gen_args, no_empty_warning=args.query.is_empty())
 
@@ -233,8 +233,8 @@ def deploy(args: cli_args.DeployOptions):
 
     deployer = Deployer(args)
     filterer = filtering.filterer_connector.get()
-    fetcher = fetcher_connector.get()
-    deploy_driver = driver_connector.get()
+    fetcher = get_fetcher()
+    deploy_driver = get_deployer()
 
     with get_loader(args, args) as loader:
         return api.deploy(

annet-0.15.3/annet/connectors.py

@@ -0,0 +1,143 @@
+import sys
+from abc import ABC, abstractmethod
+from functools import cached_property
+from importlib.metadata import entry_points
+from typing import Generic, Optional, Type, TypeVar, List, Dict, Any, Tuple
+import warnings
+from annet.lib import get_context
+
+T = TypeVar("T")
+
+
+class Connector(ABC, Generic[T]):
+    name: str
+    # legacy
+    ep_name: str
+    ep_group: str = "annet.connectors"
+    # right way just to use ep groups
+    ep_by_group_only: str = ""
+    _classes: Optional[List[Type[T]]] = None
+
+    def _get_default(self) -> Type[T]:
+        raise RuntimeError(f"{self.name} is not set")
+
+    @cached_property
+    def _entry_point(self) -> List[Type[T]]:
+        ep = load_entry_point(self.ep_group, self.ep_name)
+        if self.ep_by_group_only:
+            ep.extend(load_entry_point_new(self.ep_by_group_only))
+        return ep
+
+    def get(self, *args, **kwargs) -> T:
+        """
+        Returns connector. If more than one is registered returns random and throw warning
+        """
+        if self._classes is None:
+            self._classes = self._entry_point or [self._get_default()]
+        if not self._classes:
+            raise Exception(f"Not found registered class for group={self.ep_group}")
+        if len(self._classes) > 1:
+            warnings.warn(f"Multiple classes are registered with the group={self.ep_group} but "
+                          f"{[cls for cls in self._classes]}", UserWarning)
+        res = self._classes[0]
+        return res(*args, **kwargs)
+
+    def get_all(self, *args, **kwargs) -> List[T]:
+        if self._classes is None:
+            self._classes = self._entry_point or [self._get_default()]
+
+        return [cls(*args, **kwargs) for cls in self._classes]
+
+    def set(self, cls: Type[T]):
+        if self._classes is not None:
+            raise RuntimeError(f"Cannot reinitialize value of {self.name}")
+        self._classes = [cls]
+
+    def set_all(self, classes: List[Type[T]]):
+        if self._classes is not None:
+            raise RuntimeError(f"Cannot reinitialize value of {self.name}")
+        self._classes = list(classes)
+
+    def is_default(self) -> bool:
+        return self._classes is self._entry_point is None
+
+
+class CachedConnector(Connector[T], ABC):
+    _cache: Optional[T] = None
+
+    def get(self, *args, **kwargs) -> T:
+        assert not (args or kwargs), "Arguments forwarding is not allowed for cached connectors"
+        if self._cache is None:
+            self._cache = super().get()
+        return self._cache
+
+    def set(self, cls: Type[T]):
+        super().set(cls)
+        self._cache = None
+
+
+def load_entry_point(group: str, name: str):
+    if sys.version_info < (3, 10):
+        ep = [item for item in entry_points().get(group, []) if item.name == name]
+    else:
+        ep = entry_points(group=group, name=name)  # pylint: disable=unexpected-keyword-arg
+    if not ep:
+        return []
+    return [item.load() for item in ep]
+
+
+def load_entry_point_new(group: str) -> List:
+    if sys.version_info < (3, 10):
+        ep = [item for item in entry_points().get(group, [])]
+    else:
+        ep = entry_points(group=group)  # pylint: disable=unexpected-keyword-arg
+    if not ep:
+        return []
+    return [item.load() for item in ep]
+
+
+class AdapterWithConfig(ABC, Generic[T]):
+    @abstractmethod
+    def with_config(self, **kwargs: Dict[str, Any]) -> T:
+        pass
+
+
+class AdapterWithName(ABC):
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+
+def get_connector_from_config(config_key: str, connectors: List[Connector]) -> Tuple[Connector, Dict[str, Any]]:
+    seen: list[str] = []
+    if not connectors:
+        raise Exception("empty connectors")
+    connector = connectors[0]  # default
+    connector_params: Dict[str, Any] = {}
+    if context_storage := get_context().get(config_key):
+        adapter_name = context_storage.get("adapter", None)
+        connector_params = context_storage.get("params", {})
+        if adapter_name:
+            for con in connectors:
+                con_name = connector.__class__.__name__
+                if isinstance(con, AdapterWithName):
+                    con_name = con.name()
+                seen.append(con_name)
+                if adapter_name == con_name:
+                    connector = con
+                    break
+            else:
+                raise Exception("unknown %s %s: seen %s" % (config_key, adapter_name, seen))
+        else:
+            connector = connectors[0]
+            if len(connectors) > 1:
+                warnings.warn(f"Please specify adapter for '{config_key}'. Found more than one classes {connectors}", UserWarning)
+    else:
+        connector = connectors[0]
+        if len(connectors) > 1:
+            warnings.warn(f"Please specify '{config_key}'. Found more than one classes {connectors}", UserWarning)
+    if isinstance(connector, AdapterWithConfig):
+        connector = connector.with_config(**connector_params)
+    # return connector_params only for storage
+    # TODO: switch storage interface to AdapterWithConfig
+    return connector, connector_params

{annet-0.15.2 → annet-0.15.3}/annet/deploy.py

@@ -6,7 +6,7 @@ import itertools
 import re
 from collections import namedtuple
 from contextlib import contextmanager
-from typing import Dict, List, Optional, Type, Any, OrderedDict
+from typing import Dict, List, Optional, Any, OrderedDict, Tuple, Type
 
 from contextlog import get_logger
 
@@ -15,7 +15,7 @@ from annet.annlib.command import Command, Question, CommandList
 from annet.annlib.netdev.views.hardware import HardwareView
 from annet.annlib.rbparser.deploying import MakeMessageMatcher, Answer
 from annet.cli_args import DeployOptions
-from annet.connectors import Connector
+from annet.connectors import Connector, get_connector_from_config
 from annet.output import TextArgs
 from annet.rulebook import get_rulebook, deploying
 from annet.storage import Device
@@ -39,17 +39,23 @@ class DeployResult(_DeployResultBase):  # noqa: E302
 class _FetcherConnector(Connector["Fetcher"]):
     name = "Fetcher"
     ep_name = "deploy_fetcher"
+    ep_by_group_only = "annet.connectors.fetcher"
 
     def _get_default(self) -> Type["Fetcher"]:
-        return StubFetcher
+        # if entry points are broken, try to use direct import
+        import annet.adapters.fetchers.stub.fetcher as stub_fetcher
+        return stub_fetcher.StubFetcher
 
 
 class _DriverConnector(Connector["DeployDriver"]):
     name = "DeployDriver"
     ep_name = "deploy_driver"
+    ep_by_group_only = "annet.connectors.deployer"
 
     def _get_default(self) -> Type["DeployDriver"]:
-        return StubDeployDriver
+        # if entry points are broken, try to use direct import
+        import annet.adapters.deployers.stub.deployer as stub_deployer
+        return stub_deployer.StubDeployDriver
 
 
 fetcher_connector = _FetcherConnector()
@@ -59,7 +65,7 @@ driver_connector = _DriverConnector()
 class Fetcher(abc.ABC):
     @abc.abstractmethod
     def fetch_packages(self, devices: List[Device],
-                       processes: int = 1, max_slots: int = 0):
+                       processes: int = 1, max_slots: int = 0) -> Tuple[Dict[Device, str], Dict[Device, Any]]:
         pass
 
     @abc.abstractmethod
@@ -69,15 +75,10 @@ class Fetcher(abc.ABC):
         pass
 
 
-class StubFetcher(Fetcher):
-    def fetch_packages(self, devices: List[Device],
-                       processes: int = 1, max_slots: int = 0):
-        raise NotImplementedError()
-
-    def fetch(self, devices: List[Device],
-              files_to_download: Dict[str, List[str]] = None,
-              processes: int = 1, max_slots: int = 0):
-        raise NotImplementedError()
+def get_fetcher() -> Fetcher:
+    connectors = fetcher_connector.get_all()
+    fetcher, _ = get_connector_from_config("fetcher", connectors)
+    return fetcher
 
 
 class DeployDriver(abc.ABC):
@@ -86,11 +87,11 @@ class DeployDriver(abc.ABC):
         pass
 
     @abc.abstractmethod
-    def apply_deploy_rulebook(self, hw, cmd_paths, do_finalize=True, do_commit=True):
+    def apply_deploy_rulebook(self, hw: HardwareView, cmd_paths, do_finalize=True, do_commit=True):
         pass
 
     @abc.abstractmethod
-    def build_configuration_cmdlist(self, hw, do_finalize=True, do_commit=True):
+    def build_configuration_cmdlist(self, hw: HardwareView, do_finalize=True, do_commit=True):
         pass
 
     @abc.abstractmethod
@@ -98,18 +99,10 @@ class DeployDriver(abc.ABC):
         pass
 
 
-class StubDeployDriver(DeployDriver):
-    async def bulk_deploy(self, deploy_cmds: dict, args: DeployOptions) -> DeployResult:
-        NotImplementedError()
-
-    def apply_deploy_rulebook(self, hw, cmd_paths, do_finalize=True, do_commit=True):
-        NotImplementedError()
-
-    def build_configuration_cmdlist(self, hw, do_finalize=True, do_commit=True):
-        NotImplementedError()
-
-    def build_exit_cmdlist(self, hw):
-        raise NotImplementedError()
+def get_deployer() -> DeployDriver:
+    connectors = fetcher_connector.get_all()
+    deployer, _ = get_connector_from_config("deployer", connectors)
+    return deployer
 
 
 # ===

{annet-0.15.2 → annet-0.15.3}/annet/filtering.py

@@ -7,6 +7,7 @@ from annet.connectors import Connector
 class _FiltererConnector(Connector["Filterer"]):
     name = "Filterer"
     ep_name = "filterer"
+    ep_by_group_only = "annet.connectors.filterer"
 
     def _get_default(self) -> Type["Filterer"]:
         return NopFilterer

{annet-0.15.2 → annet-0.15.3}/annet/gen.py

@@ -30,7 +30,7 @@ from annet.annlib import jsontools
 from annet.annlib.rbparser import platform
 from annet.annlib.rbparser.acl import compile_acl_text
 from annet.cli_args import DeployOptions, GenOptions, ShowGenOptions
-from annet.deploy import fetcher_connector, scrub_config
+from annet.deploy import scrub_config, get_fetcher
 from annet.filtering import Filterer
 from annet.generators import (
     BaseGenerator,
@@ -407,7 +407,7 @@ def old_new(
     if do_files_download and config == "running":
         files_to_download = _get_files_to_download(devices, gens)
         devices_with_files = [device for device in devices if device in files_to_download]
-        fetcher = fetcher_connector.get()
+        fetcher = get_fetcher()
         fetched_packages, failed_packages = fetcher.fetch_packages(devices_with_files)
 
     ctx = OldNewDeviceContext(
@@ -808,7 +808,7 @@ def _old_resolve_running(config: str, devices: List[Device]) -> Tuple[Dict[Devic
         global live_configs  # pylint: disable=global-statement
         if live_configs is None:
             # предварительно прочесть все конфиги прямо по ssh
-            fetcher = fetcher_connector.get()
+            fetcher = get_fetcher()
             running, failed_running = fetcher.fetch(devices)
         else:
             running, failed_running = live_configs  # pylint: disable=unpacking-non-sequence
@@ -826,7 +826,7 @@ def _old_resolve_files(config: str,
         files_to_download = _get_files_to_download(devices, gens)
         devices_with_files = [device for device in devices if device in files_to_download]
         if devices_with_files:
-            fetcher = fetcher_connector.get()
+            fetcher = get_fetcher()
             downloaded_files, failed_files = fetcher.fetch(devices_with_files,
                                                            files_to_download=files_to_download)
     return downloaded_files, failed_files

{annet-0.15.2 → annet-0.15.3}/annet/hardware.py

@@ -15,6 +15,7 @@ except ImportError:
 class _HardwareConnector(Connector["HarwareProvider"]):
     name = "Hardware"
     ep_name = "hardware"
+    ep_by_group_only = "annet.connectors.hardware"
 
 
 hardware_connector = _HardwareConnector()

{annet-0.15.2 → annet-0.15.3}/annet/output.py

@@ -30,6 +30,7 @@ BLACKBOX_FILENAME = "config.cfg"
 class _DriverConnector(Connector["OutputDriver"]):
     name = "OutputDriver"
     ep_name = "output"
+    ep_by_group_only = "annet.connectors.output"
 
     def _get_default(self) -> Type["OutputDriver"]:
         return OutputDriverBasic

{annet-0.15.2 → annet-0.15.3}/annet/storage.py

@@ -1,11 +1,12 @@
 import abc
 from typing import Any, Iterable, Optional, Type, Union, Protocol, Dict
-from annet.connectors import Connector, get_context
+from annet.connectors import Connector, get_connector_from_config
 
 
 class _StorageConnector(Connector["StorageProvider"]):
-    name = "Storage"
-    ep_name = "storage"
+    name = "Storage"  # legacy
+    ep_name = "storage"  # legacy
+    ep_by_group_only = "annet.connectors.storage"
 
 
 storage_connector = _StorageConnector()
@@ -128,18 +129,6 @@ class Device(Protocol):
         pass
 
 
-def get_storage() -> (Storage, Dict[str, Any]):
+def get_storage() -> tuple[StorageProvider, Dict[str, Any]]:
     connectors = storage_connector.get_all()
-    seen: list[str] = []
-    if context_storage := get_context().get("storage"):
-        for connector in connectors:
-            con_name = connector.name()
-            seen.append(con_name)
-            if "adapter" not in context_storage:
-                raise Exception("adapter is not set in %s" % context_storage)
-            if context_storage["adapter"] == con_name:
-                return connector, context_storage.get("params", {})
-        else:
-            raise Exception("unknown storage %s: seen %s" % (context_storage["adapter"], seen))
-    else:
-        return connectors[0], {}
+    return get_connector_from_config("storage", connectors)

{annet-0.15.2 → annet-0.15.3/annet.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: annet
-Version: 0.15.2
+Version: 0.15.3
 Summary: annet
 Home-page: https://github.com/annetutil/annet
 License: MIT

{annet-0.15.2 → annet-0.15.3}/annet.egg-info/entry_points.txt

@@ -1,5 +1,8 @@
 [annet.connectors]
 storage = annet.adapters.netbox.provider:NetboxProvider
 
+[annet.connectors.storage]
+file = annet.adapters.file.provider:Provider
+
 [console_scripts]
 annet = annet.annet:main

{annet-0.15.2 → annet-0.15.3}/setup.py

@@ -39,6 +39,9 @@ if __name__ == "__main__":
           "annet.connectors": [
             "storage = annet.adapters.netbox.provider:NetboxProvider",
           ],
+          "annet.connectors.storage": [
+            "file = annet.adapters.file.provider:Provider",
+          ],
         },
         python_requires=">=3.8",
         install_requires=requirements(),

annet-0.15.2/annet/connectors.py

@@ -1,78 +0,0 @@
-import sys
-from abc import ABC
-from functools import cached_property
-from importlib.metadata import entry_points
-from typing import Generic, Optional, Type, TypeVar, List
-from annet.lib import get_context
-
-T = TypeVar("T")
-
-
-class Connector(ABC, Generic[T]):
-    name: str
-    ep_name: str
-    ep_group: str = "annet.connectors"
-    _classes: Optional[List[Type[T]]] = None
-
-    def _get_default(self) -> Type[T]:
-        raise RuntimeError(f"{self.name} is not set")
-
-    @cached_property
-    def _entry_point(self) -> List[Type[T]]:
-        return load_entry_point(self.ep_group, self.ep_name)
-
-    def get(self, *args, **kwargs) -> T:
-        if self._classes is None:
-            self._classes = self._entry_point or [self._get_default()]
-        if len(self._classes) > 1:
-            raise RuntimeError(
-                f"Multiple classes are registered with the same "
-                f"group={self.ep_group} and name={self.ep_name}: "
-                f"{[cls for cls in self._classes]}",
-            )
-
-        res = self._classes[0]
-        return res(*args, **kwargs)
-
-    def get_all(self, *args, **kwargs) -> List[T]:
-        if self._classes is None:
-            self._classes = self._entry_point or [self._get_default()]
-
-        return [cls(*args, **kwargs) for cls in self._classes]
-
-    def set(self, cls: Type[T]):
-        if self._classes is not None:
-            raise RuntimeError(f"Cannot reinitialize value of {self.name}")
-        self._classes = [cls]
-
-    def set_all(self, classes: List[Type[T]]):
-        if self._classes is not None:
-            raise RuntimeError(f"Cannot reinitialize value of {self.name}")
-        self._classes = list(classes)
-
-    def is_default(self) -> bool:
-        return self._classes is self._entry_point is None
-
-
-class CachedConnector(Connector[T], ABC):
-    _cache: Optional[T] = None
-
-    def get(self, *args, **kwargs) -> T:
-        assert not (args or kwargs), "Arguments forwarding is not allowed for cached connectors"
-        if self._cache is None:
-            self._cache = super().get()
-        return self._cache
-
-    def set(self, cls: Type[T]):
-        super().set(cls)
-        self._cache = None
-
-
-def load_entry_point(group: str, name: str):
-    if sys.version_info < (3, 10):
-        ep = [item for item in entry_points().get(group, []) if item.name == name]
-    else:
-        ep = entry_points(group=group, name=name)  # pylint: disable=unexpected-keyword-arg
-    if not ep:
-        return None
-    return [item.load() for item in ep]