gnetcli-adapter 0.0.2__tar.gz

gnetcli_adapter-0.0.2/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.1
+Name: gnetcli_adapter
+Version: 0.0.2
+Summary: Gnetcli-server adapter for Annet
+Author-email: Aleksandr Balezin <gescheit12@gmail.com>
+Requires-Python: >=3.8.1
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: annet
+Requires-Dist: gnetclisdk>=1.0.2
+Requires-Dist: pydantic_settings
+Requires-Dist: pydantic
+Requires-Dist: bandit[toml]==1.7.5 ; extra == "test"
+Requires-Dist: black==23.3.0 ; extra == "test"
+Requires-Dist: check-manifest==0.49 ; extra == "test"
+Requires-Dist: flake8-bugbear==23.5.9 ; extra == "test"
+Requires-Dist: flake8-docstrings ; extra == "test"
+Requires-Dist: flake8-formatter_junit_xml ; extra == "test"
+Requires-Dist: flake8 ; extra == "test"
+Requires-Dist: flake8-pyproject ; extra == "test"
+Requires-Dist: pre-commit==3.3.1 ; extra == "test"
+Requires-Dist: pylint==2.17.4 ; extra == "test"
+Requires-Dist: pylint_junit ; extra == "test"
+Requires-Dist: pytest-cov==4.0.0 ; extra == "test"
+Requires-Dist: pytest-mock<3.10.1 ; extra == "test"
+Requires-Dist: pytest-runner ; extra == "test"
+Requires-Dist: pytest==7.3.1 ; extra == "test"
+Requires-Dist: pytest-github-actions-annotate-failures ; extra == "test"
+Requires-Dist: shellcheck-py==0.9.0.2 ; extra == "test"
+Project-URL: Documentation, https://github.com/annetutil/gnetcli_adapter
+Project-URL: Source, https://github.com/annetutil/gnetcli_adapter
+Project-URL: Tracker, https://github.com/annetutil/gnetcli_adapter/issues
+Provides-Extra: test
+
+# gnetcli_adapter
+This package provides deployer and fetcher adapters for Annet
+

gnetcli_adapter-0.0.2/README.md

@@ -0,0 +1,2 @@
+# gnetcli_adapter
+This package provides deployer and fetcher adapters for Annet

gnetcli_adapter-0.0.2/pyproject.toml

@@ -0,0 +1,309 @@
+[build-system]
+requires = ["flit_core >=2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name =  "gnetcli_adapter"
+dependencies = [
+  "annet",
+  "gnetclisdk>=1.0.2",
+  "pydantic_settings",
+  "pydantic"
+]
+authors = [
+    {name = "Aleksandr Balezin", email = "gescheit12@gmail.com"},
+]
+description = "Gnetcli-server adapter for Annet"
+readme = "README.md"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11"
+]
+requires-python = ">=3.8.1"
+dynamic = ["version"]
+
+[project.entry-points."annet.adapters"]
+deploy_fetcher = "gnetcli_adapter.gnetcli_adapter:GnetcliFetcher"
+deploy_driver = "gnetcli_adapter.gnetcli_adapter:GnetcliDeployer"
+
+[project.optional-dependencies]
+test = [
+    "bandit[toml]==1.7.5",
+    "black==23.3.0",
+    "check-manifest==0.49",
+    "flake8-bugbear==23.5.9",
+    "flake8-docstrings",
+    "flake8-formatter_junit_xml",
+    "flake8",
+    "flake8-pyproject",
+    "pre-commit==3.3.1",
+    "pylint==2.17.4",
+    "pylint_junit",
+    "pytest-cov==4.0.0",
+    "pytest-mock<3.10.1",
+    "pytest-runner",
+    "pytest==7.3.1",
+    "pytest-github-actions-annotate-failures",
+    "shellcheck-py==0.9.0.2"
+]
+
+[project.urls]
+Documentation = "https://github.com/annetutil/gnetcli_adapter"
+Source = "https://github.com/annetutil/gnetcli_adapter"
+Tracker = "https://github.com/annetutil/gnetcli_adapter/issues"
+
+[tool.flit.module]
+name = "gnetcli_adapter"
+
+[tool.bandit]
+exclude_dirs = ["build","dist","tests","scripts"]
+number = 4
+recursive = true
+targets = "src"
+
+[tool.black]
+line-length = 120
+fast = true
+
+[tool.coverage.run]
+branch = true
+
+[tool.coverage.report]
+fail_under = 100
+
+[tool.flake8]
+max-line-length = 120
+select = "F,E,W,B,B901,B902,B903"
+exclude = [
+    ".eggs",
+    ".git",
+    ".tox",
+    "nssm",
+    "obj",
+    "out",
+    "packages",
+    "pywin32",
+    "tests",
+    "swagger_client"
+]
+ignore = [
+    "E722",
+    "B001",
+    "W503",
+    "E203"
+]
+
+[tool.pyright]
+include = ["src"]
+exclude = [
+    "**/__pycache__",
+]
+venv = "env37"
+
+reportMissingImports = true
+reportMissingTypeStubs = false
+
+pythonVersion = "3.7"
+pythonPlatform = "Linux"
+
+executionEnvironments = [
+  { root = "src" }
+]
+
+[tool.pytest.ini_options]
+addopts = "--cov-report xml:coverage.xml --cov src --cov-fail-under 0 --cov-append -m 'not integration'"
+pythonpath = [
+  "src"
+]
+testpaths = "tests"
+junit_family = "xunit2"
+markers = [
+    "integration: marks as integration test",
+    "notebooks: marks as notebook test",
+    "slow: marks tests as slow",
+    "unit: fast offline tests",
+]
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+envlist = py, integration, all
+
+[testenv]
+commands =
+    pytest -m "not integration" {posargs}
+
+[testenv:integration]
+commands =
+    pytest -m "integration" {posargs}
+
+[testenv:all]
+extras = all
+commands =
+    pytest {posargs}
+"""
+
+[tool.pylint]
+extension-pkg-whitelist= [
+    "numpy",
+    "torch",
+    "cv2",
+    "pyodbc",
+    "pydantic",
+    "ciso8601",
+    "netcdf4",
+    "scipy"
+]
+ignore="CVS"
+ignore-patterns="test.*?py,conftest.py"
+init-hook='import sys; sys.setrecursionlimit(8 * sys.getrecursionlimit())'
+jobs=0
+limit-inference-results=100
+persistent="yes"
+suggestion-mode="yes"
+unsafe-load-any-extension="no"
+
+[tool.pylint.'MESSAGES CONTROL']
+enable="c-extension-no-member"
+disable="missing-function-docstring,global-statement,wrong-import-order,missing-class-docstring,too-many-arguments,too-many-positional-arguments"
+
+[tool.pylint.'REPORTS']
+evaluation="10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)"
+output-format="text"
+reports="no"
+score="yes"
+
+[tool.pylint.'REFACTORING']
+max-nested-blocks=5
+never-returning-functions="sys.exit"
+
+[tool.pylint.'BASIC']
+argument-naming-style="snake_case"
+attr-naming-style="snake_case"
+bad-names= [
+    "foo",
+    "bar"
+]
+class-attribute-naming-style="any"
+class-naming-style="PascalCase"
+const-naming-style="UPPER_CASE"
+docstring-min-length=-1
+function-naming-style="snake_case"
+good-names= [
+    "i",
+    "j",
+    "k",
+    "ex",
+    "Run",
+    "_"
+]
+include-naming-hint="yes"
+inlinevar-naming-style="any"
+method-naming-style="snake_case"
+module-naming-style="any"
+no-docstring-rgx="^_"
+property-classes="abc.abstractproperty"
+variable-naming-style="snake_case"
+
+[tool.pylint.'FORMAT']
+ignore-long-lines="^\\s*(# )?.*['\"]?<?https?://\\S+>?"
+indent-after-paren=4
+indent-string='    '
+max-line-length=120
+max-module-lines=1000
+single-line-class-stmt="no"
+single-line-if-stmt="no"
+
+[tool.pylint.'LOGGING']
+logging-format-style="old"
+logging-modules="logging"
+
+[tool.pylint.'MISCELLANEOUS']
+notes= [
+    "FIXME",
+    "XXX",
+    "TODO"
+]
+
+[tool.pylint.'SIMILARITIES']
+ignore-comments="yes"
+ignore-docstrings="yes"
+ignore-imports="yes"
+min-similarity-lines=7
+
+[tool.pylint.'SPELLING']
+max-spelling-suggestions=4
+spelling-store-unknown-words="no"
+
+[tool.pylint.'STRING']
+check-str-concat-over-line-jumps="no"
+
+[tool.pylint.'TYPECHECK']
+contextmanager-decorators="contextlib.contextmanager"
+generated-members="numpy.*,np.*.sql.functions,collect_list"
+ignore-mixin-members="yes"
+ignore-none="yes"
+ignore-on-opaque-inference="yes"
+ignored-classes="optparse.Values,thread._local,_thread._local,numpy,torch,swagger_client"
+ignored-modules="numpy,torch,swagger_client,netCDF4,scipy"
+missing-member-hint="yes"
+missing-member-hint-distance=1
+missing-member-max-choices=1
+
+[tool.pylint.'VARIABLES']
+additional-builtins="dbutils"
+allow-global-unused-variables="yes"
+callbacks= [
+    "cb_",
+    "_cb"
+]
+dummy-variables-rgx="_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_"
+ignored-argument-names="_.*|^ignored_|^unused_"
+init-import="no"
+redefining-builtins-modules="six.moves,past.builtins,future.builtins,builtins,io"
+
+[tool.pylint.'CLASSES']
+defining-attr-methods= [
+    "__init__",
+    "__new__",
+    "setUp",
+    "__post_init__"
+]
+exclude-protected= [
+    "_asdict",
+    "_fields",
+    "_replace",
+    "_source",
+    "_make"
+]
+valid-classmethod-first-arg="cls"
+valid-metaclass-classmethod-first-arg="cls"
+
+[tool.pylint.'DESIGN']
+max-args=5
+max-attributes=7
+max-bool-expr=5
+max-branches=12
+max-locals=15
+max-parents=7
+max-public-methods=20
+max-returns=6
+max-statements=50
+min-public-methods=2
+
+[tool.pylint.'IMPORTS']
+allow-wildcard-with-all="no"
+analyse-fallback-blocks="no"
+deprecated-modules="optparse,tkinter.tix"
+
+[tool.pylint.'EXCEPTIONS']
+overgeneral-exceptions= [
+    "BaseException",
+    "Exception"
+]

gnetcli_adapter-0.0.2/src/gnetcli_adapter/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.2"

gnetcli_adapter-0.0.2/src/gnetcli_adapter/gnetcli_adapter.py

@@ -0,0 +1,293 @@
+import asyncio
+import json
+import subprocess
+import time
+
+from annet.deploy import DeployDriver, DeployOptions, DeployResult, apply_deploy_rulebook
+from annet.annlib.command import Command, CommandList
+from annet.annlib.netdev.views.hardware import HardwareView
+from annet.rulebook import common
+
+from annet.deploy import Fetcher, AdapterWithConfig, AdapterWithName
+from typing import Dict, List, Any, Optional, Tuple
+from annet.storage import Device
+from gnetclisdk.client import Credentials, Gnetcli, HostParams
+from pydantic import Field, field_validator, FieldValidationInfo
+from pydantic_core import PydanticUndefined
+from pydantic_settings import BaseSettings, SettingsConfigDict
+import base64
+import logging
+import threading
+import atexit
+
+breed_to_device = {
+    "routeros": "ros",
+}
+_local_gnetcli: Optional[threading.Thread] = None
+_local_gnetcli_p: Optional[subprocess.Popen] = None
+_local_gnetcli_url: Optional[str] = None
+LOG_FORMAT = "%(asctime)s - l:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s"
+DATE_FMT = "%Y-%m-%d %H:%M:%S"
+_logger = logging.getLogger(__name__)
+GNETCLI_SERVER = "/usr/bin/server"
+
+
+class AppSettings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="gnetcli_", validate_assignment=True)
+
+    server: str = Field(default="localhost:50051")
+    insecure_grpc: bool = Field(default=True)
+    login: str = Field(default="")
+    password: str = Field(default="")
+    dev_login: str = Field(default="")
+    dev_password: str = Field(default="")
+
+    def make_credentials(self) -> Credentials:
+        return Credentials(self.dev_login, self.dev_password)
+
+    @field_validator("*", mode="before")
+    @classmethod
+    def not_none(cls, value: Any, info: FieldValidationInfo):
+        # NOTE: All fields that are optional for values, will assume the value in
+        # "default" (if defined in "Field") if "None" is informed as "value". That
+        # is, "None" is never assumed if passed as a "value".
+        if (
+            cls.model_fields[info.field_name].get_default() is not PydanticUndefined
+            and not cls.model_fields[info.field_name].is_required()
+            and value is None
+        ):
+            return cls.model_fields[info.field_name].get_default()
+        return value
+
+
+async def get_config(breed: str) -> List[str]:
+    if breed == "routeros":
+        return ["/export"]
+    raise Exception("unknown breed")
+
+
+def check_gnetcli_server():
+    global _local_gnetcli
+    if not _local_gnetcli:
+        t = threading.Thread(target=run_gnetcli_server, args=())
+        t.daemon = True
+        t.start()
+        time.sleep(1)
+        _local_gnetcli = t
+    if _local_gnetcli_p is None:
+        raise Exception("server failed")
+
+
+def cleanup():
+    if _local_gnetcli_p is not None:
+        _local_gnetcli_p.kill()
+
+
+atexit.register(cleanup)
+
+
+def run_gnetcli_server():
+    global _local_gnetcli_p
+    global _local_gnetcli_url
+    _logger.info("starting gnetcli server")
+    try:
+        proc = subprocess.Popen(
+            [GNETCLI_SERVER, "--conf-file", "-"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            bufsize=1,
+            universal_newlines=True,
+        )
+    except Exception as e:
+        logging.exception("server exec error %s", e)
+        raise
+    proc.stdin.write(
+        """
+logging:
+  level: debug
+  json: true
+port: 0
+    """
+    )
+    proc.stdin.close()
+    _local_gnetcli_p = proc
+    while True:
+        output = proc.stderr.readline()
+        if output == "" and proc.poll() is not None:
+            break
+        if output:
+            _logger.debug("gnetcli output: %s", output.strip())
+            if _local_gnetcli_url is None:
+                try:
+                    data = json.loads(output)
+                except Exception:
+                    pass
+                else:
+                    if data.get("msg") == "init tcp socket":
+                        _local_gnetcli_url = data.get("address")
+                    if data.get("level") == "panic":
+                        _logger.error("gnetcli error %s", data)
+    _logger.debug("set gnetcli server exit code %s", proc.returncode)
+    rc = proc.poll()
+    return rc
+
+
+class GnetcliFetcher(Fetcher, AdapterWithConfig, AdapterWithName):
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        login: Optional[str] = None,
+        password: Optional[str] = None,
+        dev_login: Optional[str] = None,
+        dev_password: Optional[str] = None,
+    ):
+        if not url:
+            check_gnetcli_server()
+            if _local_gnetcli_url is None:
+                _logger.info("waiting for _local_gnetcli_url appears")
+                start = time.monotonic()
+                while time.monotonic() - start < 5:
+                    if _local_gnetcli_p is not None and _local_gnetcli_p.returncode is not None:
+                        raise Exception("gnetcli server died with code %s" % _local_gnetcli_p.returncode)
+                    if _local_gnetcli_url is not None:
+                        break
+
+        self.conf = AppSettings(
+            login=login, password=password, dev_login=dev_login, dev_password=dev_password, server=_local_gnetcli_url
+        )
+        auth_token = (
+            base64.b64encode(b"%s:%s" % (self.conf.login.encode(), self.conf.password.encode())).strip().decode()
+        )
+        auth_token = f"Basic {auth_token}"
+        self.api = Gnetcli(
+            server=self.conf.server,
+            auth_token=auth_token,
+            insecure_grpc=self.conf.insecure_grpc,
+            user_agent="annet",
+        )
+
+    def name(self) -> str:
+        return "gnetcli"
+
+    def with_config(self, **kwargs: Dict[str, Any]) -> Fetcher:
+        return GnetcliFetcher(**kwargs)
+
+    def fetch_packages(self, devices: List[Device], processes: int = 1, max_slots: int = 0):
+        if not devices:
+            return {}, {}
+        raise NotImplementedError()
+
+    def fetch(
+        self,
+        devices: List[Device],
+        files_to_download: Dict[str, List[str]] = None,
+        processes: int = 1,
+        max_slots: int = 0,
+    ) -> Tuple[Dict[Device, str], Dict[Device, Any]]:
+        return asyncio.run(self.afetch(devices=devices))
+
+    async def afetch(self, devices: List[Device]):
+        running = {}
+        failed_running = {}
+        for device in devices:
+            try:
+                dev_res = await self.afetch_dev(device=device)
+            except Exception as e:
+                failed_running[device] = e
+            else:
+                running[device] = dev_res
+        return running, failed_running
+
+    async def afetch_dev(self, device: Device) -> str:
+        if device.breed not in breed_to_device:
+            raise Exception("unknown breed for gnetcli")
+        device_cls = breed_to_device[device.breed]
+
+        cmds = await get_config(breed=device.breed)
+        dev_result = []
+        for cmd in cmds:
+            res = await self.api.cmd(
+                hostname=device.fqdn,
+                cmd=cmd,
+                host_params=HostParams(
+                    credentials=self.conf.make_credentials(),
+                    device=device_cls,
+                ),
+            )
+            if res.status != 0:
+                raise Exception("cmd error %s" % res)
+            dev_result.append(res.out)
+        return b"\n".join(dev_result).decode()
+
+
+class GnetcliDeployer(DeployDriver, AdapterWithConfig, AdapterWithName):
+    def __init__(
+        self,
+        login: Optional[str] = None,
+        password: Optional[str] = None,
+        dev_login: Optional[str] = None,
+        dev_password: Optional[str] = None,
+    ):
+        self.conf = AppSettings(login=login, password=password, dev_login=dev_login, dev_password=dev_password)
+        auth_token = (
+            base64.b64encode(b"%s:%s" % (self.conf.login.encode(), self.conf.password.encode())).strip().decode()
+        )
+        auth_token = f"Basic {auth_token}"
+        self.api = Gnetcli(
+            server=self.conf.server,
+            auth_token=auth_token,
+            insecure_grpc=self.conf.insecure_grpc,
+            user_agent="annet",
+        )
+
+    def name(self) -> str:
+        return "gnetcli"
+
+    def with_config(self, **kwargs: Dict[str, Any]) -> DeployDriver:
+        return GnetcliDeployer(**kwargs)
+
+    async def bulk_deploy(self, deploy_cmds: Dict[Device, CommandList], args: DeployOptions) -> DeployResult:
+        deploy_items = deploy_cmds.items()
+        result = await asyncio.gather(*[asyncio.Task(self.deploy(device, cmds, args)) for device, cmds in deploy_items])
+        res = DeployResult(hostnames=[], results={}, durations={}, original_states={})
+        res.add_results(results={dev.fqdn: dev_res for (dev, _), dev_res in zip(deploy_items, result)})
+        return res
+
+    async def deploy(self, device: Device, cmds: CommandList, args: DeployOptions) -> str:
+        device_cls = breed_to_device[device.breed]
+        async with self.api.cmd_session(hostname=device.fqdn) as sess:
+            result = []
+            for cmd in cmds:
+                res = await sess.cmd(
+                    cmd=cmd.cmd,
+                    cmd_timeout=cmd.timeout,
+                    host_params=HostParams(
+                        credentials=self.conf.make_credentials(),
+                        device=device_cls,
+                    ),
+                )
+                if res.status != 0:
+                    raise Exception("cmd %s error %s status %s", cmd, res.err, res.status)
+                result.append(res)
+            return result
+
+    def apply_deploy_rulebook(
+        self,
+        hw: HardwareView,
+        cmd_paths: Dict[Tuple[str, ...], Dict[str, Any]],
+        do_finalize: bool = True,
+        do_commit: bool = True,
+    ):
+        res = apply_deploy_rulebook(hw=hw, cmd_paths=cmd_paths, do_finalize=do_finalize, do_commit=do_commit)
+        return res
+
+    def build_configuration_cmdlist(self, hw: HardwareView, do_finalize: bool = True, do_commit: bool = True):
+        res = common.apply(hw, do_commit=do_commit, do_finalize=do_finalize)
+        return res
+
+    def build_exit_cmdlist(self, hw: HardwareView) -> CommandList:
+        ret = CommandList()
+        if hw.Huawei:
+            ret.add_cmd(Command("quit", suppress_eof=True))
+        return ret