nbforager 0.2.2__tar.gz → 0.4.1__tar.gz

{nbforager-0.2.2 → nbforager-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nbforager
-Version: 0.2.2
+Version: 0.4.1
 Summary: Python package designed to assist in working with the Netbox REST API. The filter parameters are identical to those in the Web UI filter form. It replaces brief data with full information, and Netbox objects are represented as a recursive multidimensional dictionary.
 Home-page: https://github.com/vladimirs-git/nbforager
 License: Apache-2.0
@@ -20,18 +20,25 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: test
 Requires-Dist: ciscoconfparse (>=1.9,<2.0)
-Requires-Dist: netports (>=0.12.1,<0.13.0)
+Requires-Dist: netports (>=0.14,<1.0.0)
 Requires-Dist: pydantic (>=2,<3)
 Requires-Dist: pynetbox (>=7.3.3,<8.0.0)
 Requires-Dist: requests (>=2,<3)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: tomli (==2.0.1)
-Requires-Dist: vhelpers (>=0.1,<0.2)
+Requires-Dist: vhelpers (>=0.3,<1.0.0)
 Project-URL: Bug Tracker, https://github.com/vladimirs-git/nbforager/issues
-Project-URL: Download URL, https://github.com/vladimirs-git/nbforager/archive/refs/tags/0.2.2.tar.gz
+Project-URL: Download URL, https://github.com/vladimirs-git/nbforager/archive/refs/tags/0.4.1.tar.gz
 Project-URL: Repository, https://github.com/vladimirs-git/nbforager
 Description-Content-Type: text/x-rst
 
+
+.. image:: https://img.shields.io/pypi/v/netports.svg
+   :target: https://pypi.python.org/pypi/netports
+.. image:: https://img.shields.io/badge/Python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg
+   :target: https://pypi.python.org/pypi/logger-color
+
+
 nbforager
 =========
 
@@ -51,6 +58,12 @@ Fully documented on `Read the Docs`_.
 
 ----------------------------------------------------------------------------------------
 
+Requirements
+============
+
+Python >=3.8,<3.12
+
+
 Quickstart
 ==========
 

{nbforager-0.2.2 → nbforager-0.4.1}/README.rst

@@ -1,3 +1,10 @@
+
+.. image:: https://img.shields.io/pypi/v/netports.svg
+   :target: https://pypi.python.org/pypi/netports
+.. image:: https://img.shields.io/badge/Python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg
+   :target: https://pypi.python.org/pypi/logger-color
+
+
 nbforager
 =========
 
@@ -17,6 +24,12 @@ Fully documented on `Read the Docs`_.
 
 ----------------------------------------------------------------------------------------
 
+Requirements
+============
+
+Python >=3.8,<3.12
+
+
 Quickstart
 ==========
 

{nbforager-0.2.2 → nbforager-0.4.1}/nbforager/__init__.py

@@ -1,5 +1,6 @@
 """nbforager."""
 
+from nbforager.foragers.joiner import Joiner
 from nbforager.nb_api import NbApi
 from nbforager.nb_forager import NbForager
 from nbforager.nb_tree import NbTree
@@ -8,10 +9,11 @@ from nbforager.parser.nb_parser import NbParser
 from nbforager.parser.nb_value import NbValue
 
 __all__ = [
+    "Joiner",
     "NbApi",
-    "NbParser",
     "NbCustom",
     "NbForager",
+    "NbParser",
     "NbTree",
     "NbValue",
 ]

{nbforager-0.2.2 → nbforager-0.4.1}/nbforager/api/base_c.py

@@ -70,9 +70,9 @@ class BaseC:
         "default_get",
         "loners",
     ]
-    _reserved_keys: DLStr = {
+    _extra_keys: DLStr = {
         "ipam/": [
-            # ipam aggregates, prefixes, ip_addresses
+            # ipam/aggregates, ipam/prefixes, ipam/ip_addresses
             "_ipv4",
             "_aggregate",
             "_super_prefix",
@@ -85,6 +85,7 @@ class BaseC:
             "nbnets__subnets",
         ],
         "dcim/devices/": [
+            # UI interfaces
             "_interfaces",
             "_front_ports",
             "_rear_ports",
@@ -95,6 +96,8 @@ class BaseC:
             "_module_bays",
             "_device_bays",
             "_inventory_items",
+            # UI virtual_chassis
+            "_vc_members",
         ],
         "dcim/interfaces/": [
             "_ip_addresses",
@@ -106,6 +109,11 @@ class BaseC:
             "_ip_addresses",
         ],
     }
+    """Extra keys are reserved for internal usage by this package.
+
+    The key represents the app/model path, while the value is the key name that can be used 
+    to store additional data generated by this package.
+    """
 
     def __init__(self, **kwargs):
         """Init BaseC.
@@ -149,9 +157,10 @@ class BaseC:
             session timeout reached. Default is `10`.
 
         :param bool strict: When querying objects by tag, if there are no tags present,
-            the Netbox API response returns a status_code=400. True - ConnectionError is
-            raised when status_code=400. False - a warning message is logged and an
-            empty list is returned with status_code=200. Default is `False`.
+            the Netbox API response returns a status_code=400.
+            True - ConnectionError is raised when status_code=400.
+            False - WARNING message is logged and an empty list is returned with status_code=200.
+            Default is `False`.
 
         :param bool extended_get: True - Extend filtering parameters in GET request,
             ``{parameter}`` can be used instead of ``{parameter}_id``. Default is `True`.
@@ -172,9 +181,9 @@ class BaseC:
         self.threads: int = _init_threads(**kwargs)
         self.interval: float = float(kwargs.get("interval") or 0.0)
         # Errors processing
-        self.timeout: float = float(kwargs.get("timeout") or 60)
+        self.timeout: int = int(kwargs.get("timeout") or 60)
         self.max_retries: int = int(kwargs.get("max_retries") or 0)
-        self.sleep: float = float(kwargs.get("sleep") or 10)
+        self.sleep: int = int(kwargs.get("sleep") or 10)
         self.strict: bool = bool(kwargs.get("strict"))
         # Settings
         self.extended_get: bool = bool(kwargs.get("extended_get"))
@@ -308,6 +317,9 @@ class BaseC:
         :return: Netbox objects. Update self _results.
         """
         offset = 0
+        if offsets := params_d.get("offset"):
+            offset = int(offsets[0])
+
         max_limit: int = self._set_limit(params_d)
         params_l: LParam = vparam.from_dict(params_d)
 
@@ -343,12 +355,12 @@ class BaseC:
         :return: Max limit value, update params_d["limit"] value
         """
         limit = 0
-        if limit_ := vdict.pop("limit", params_d) or []:
+        if limit_ := vdict.pop(params_d, key="limit") or []:
             limit = int(limit_[0])
         if not limit:
             limit = self.limit
         max_limit = 0
-        if max_limit_ := vdict.pop("max_limit", params_d) or []:
+        if max_limit_ := vdict.pop(params_d, key="max_limit") or []:
             max_limit = int(max_limit_[0])
         if max_limit and max_limit < limit:
             limit = max_limit
@@ -477,7 +489,7 @@ class BaseC:
                     url=url,
                     headers=self._headers(),
                     verify=self.verify,
-                    timeout=self.timeout,
+                    timeout=float(self.timeout),
                 )
             except ReadTimeout:
                 attempts = f"{counter} of {self.max_retries}"
@@ -493,6 +505,7 @@ class BaseC:
 
             if response.ok:
                 return response
+
             msg = self._msg_status_code(response)
             if self._is_status_code_5xx(response):
                 raise ConnectionError(f"Netbox server error: {msg}.")
@@ -536,8 +549,8 @@ class BaseC:
         }
         return headers
 
-    def _check_reserved_keys(self, items: LDAny) -> None:
-        """Check if reserved keys are present in the data.
+    def _check_extra_keys(self, items: LDAny) -> None:
+        """Check if extra keys are present in the data.
 
         The Netbox REST API returns the object as a dictionary.
         NbForager inject extra key/value pairs into Netbox object.
@@ -548,10 +561,10 @@ class BaseC:
         """
         for data in items:
             url_o: ParseResult = urllib.parse.urlparse(data["url"])
-            for path, reserved_keys in self._reserved_keys.items():
+            for path, extra_keys in self._extra_keys.items():
                 if url_o.path.find(path) > -1:
-                    if keys := set(reserved_keys).intersection(data):
-                        raise NbApiError(f"NbForager reserved {keys=} detected in {self.url}.")
+                    if keys := set(extra_keys).intersection(data):
+                        raise NbApiError(f"NbForager extra {keys=} detected in {self.url}.")
 
     def _slice_params_counters(self, results: LDAny) -> LDAny:
         """Generate sliced parameters based on counts in results.

{nbforager-0.2.2 → nbforager-0.4.1}/nbforager/api/connector.py

@@ -23,9 +23,9 @@ class Connector(BaseC):
     # ============================= methods ==============================
 
     def create(self, **kwargs) -> Response:
-        """Create object in Netbox.
+        """Create an object in Netbox.
 
-        :param kwargs: Parameters of new object to create.
+        :param kwargs: Parameters for creating a new object.
 
         :return: Session response.
 
@@ -43,11 +43,11 @@ class Connector(BaseC):
         return response
 
     def create_d(self, **kwargs) -> DAny:
-        """Create object in Netbox.
+        """Create an object in Netbox.
 
-        :param kwargs: Data of new object to create.
+        :param kwargs: Parameters for creating a new object.
 
-        :return: Data of newly crated object.
+        :return: Data of newly created object.
         :rtype: dict
         """
         response: Response = self.create(**kwargs)
@@ -59,7 +59,7 @@ class Connector(BaseC):
 
     # noinspection PyShadowingBuiltins
     def delete(self, id: int) -> Response:  # pylint: disable=redefined-builtin
-        """Delete object in Netbox.
+        """Delete an object in Netbox.
 
         :param id: Object ID.
         :type id: int
@@ -70,6 +70,8 @@ class Connector(BaseC):
             - <Response [404]> Object not found.
         :rtype: Response
         """
+        if not id:
+            raise ValueError("id is required.")
         response: Response = self._session.delete(
             url=f"{self.url}{id}",
             headers=self._headers(),
@@ -118,17 +120,17 @@ class Connector(BaseC):
         """
         params_ld: LDList = self._validate_params(**kwargs)
         items: LDAny = self._query_params_ld(params_ld)
-        self._check_reserved_keys(items=items)
+        self._check_extra_keys(items=items)
         return items
 
     # noinspection PyIncorrectDocstring
     def update(self, **kwargs) -> Response:
-        """Update object in Netbox.
+        """Update an object in Netbox.
 
         :param id: Netbox object id to update.
         :type id: int
 
-        :param kwargs: Parameters to update object in Netbox.
+        :param kwargs: Parameters to update an object in Netbox.
 
         :return: Session response.
 
@@ -136,9 +138,9 @@ class Connector(BaseC):
             - <Response [400]> Invalid data.
         :rtype: Response
         """
-        id_ = vdict.pop("id", kwargs)
+        id_ = vdict.pop(kwargs, key="id")
         if not id_:
-            raise ValueError("id expected in the data.")
+            raise ValueError("id is required in the data.")
 
         response: Response = self._session.patch(
             url=f"{self.url}{id_}/",
@@ -151,12 +153,12 @@ class Connector(BaseC):
 
     # noinspection PyIncorrectDocstring
     def update_d(self, **kwargs) -> DAny:
-        """Update object in Netbox.
+        """Update an object in Netbox.
 
         :param id: Netbox object id to update.
         :type id: int
 
-        :param kwargs: Parameters to update object in Netbox.
+        :param kwargs: Parameters to update an object in Netbox.
 
         :return: Data of updated object.
         :rtype: dict

{nbforager-0.2.2 → nbforager-0.4.1}/nbforager/foragers/forager.py

@@ -4,20 +4,20 @@
 
 from __future__ import annotations
 
-import logging
 import time
 from queue import Queue
 from threading import Thread
 from urllib.parse import urlparse, parse_qs
 
-from vhelpers import vlist, vstr
+from vhelpers import vstr
 
 from nbforager import helpers as h
+from nbforager import nb_tree
 from nbforager.nb_api import NbApi
-from nbforager.nb_tree import NbTree, missed_urls
-from nbforager.parser.nb_parser import NbParser
+from nbforager.nb_tree import NbTree
+from nbforager.parser import nb_parser
 from nbforager.py_tree import PyTree
-from nbforager.types_ import LDAny, DiDAny, LStr, LT2StrDAny, DList, LDList, TLists, DiAny
+from nbforager.types_ import LDAny, DiDAny, LStr, LT2StrDAny, DList, LDList, DiAny, SInt, DAny
 
 
 class Forager:
@@ -86,7 +86,10 @@ class Forager:
         :return: None. Update self object.
         """
         # Query main data
-        nb_objects: LDAny = self._get_root_data_from_netbox(**kwargs)
+        kwargs_: DAny = self._delete_existing_nested_ids(**kwargs)
+        if kwargs and not kwargs_:
+            return
+        nb_objects: LDAny = self._get_root_data_from_netbox(nested=nested, **kwargs_)
         if not nested:
             return
         urls: LStr = self._collect_nested_urls(nb_objects)
@@ -104,7 +107,7 @@ class Forager:
         # loop
         else:
             for url in urls:
-                app, model, _ = h.split_url(url)
+                app, model, _ = h.url_to_ami_items(url)
                 path = f"{app}/{model}/"
                 connector = self._get_connector(path)
                 params_d: DList = parse_qs(urlparse(url).query)
@@ -134,7 +137,7 @@ class Forager:
 
         :return: Filtered Netbox objects.
         """
-        return _find(objects=list(self.root_d.values()), **kwargs)
+        return nb_parser.find_objects(objects=list(self.root_d.values()), **kwargs)
 
     def find_rse(self, role: str = "", site: str = "", env: str = "", **kwargs) -> LDAny:
         """Find Netbox objects in NbForager.tree by Role-Sile-Env finding parameters.
@@ -155,7 +158,7 @@ class Forager:
         }
         params = {k: v for k, v in params.items() if v}
         kwargs.update(params)
-        return _find(objects=list(self.tree_d.values()), **kwargs)
+        return nb_parser.find_objects(objects=list(self.tree_d.values()), **kwargs)
 
     def find_tree(self, **kwargs) -> LDAny:
         """Find Netbox objects in NbForager.tree by extended finding parameters.
@@ -167,48 +170,50 @@ class Forager:
 
         :return: Filtered Netbox objects.
         """
-        return _find(objects=list(self.tree_d.values()), **kwargs)
+        return nb_parser.find_objects(objects=list(self.tree_d.values()), **kwargs)
 
     # ============================= helpers ==============================
 
-    def _get_root_data_from_netbox(self, **kwargs) -> LDAny:
+    def _delete_existing_nested_ids(self, **kwargs) -> DAny:
+        """Delete the IDs of objects that are already present in the tree and nested=True.
+
+        Delete only if kwargs["id"] is a list, ignore other data types.
+
+        :param kwargs: Filtering parameters.
+
+        :return: None. Update IDs in kwargs.
+        """
+        if list(kwargs) != ["id"]:
+            return kwargs
+        if not isinstance(kwargs["id"], list):
+            return kwargs
+        ids: SInt = set(kwargs["id"])
+        existing_ids: SInt = {d["id"] for d in self.find_root(_nested=True, **kwargs)}
+        if new_ids := sorted(set(ids).difference(existing_ids)):
+            return {"id": new_ids}
+        return {}
+
+    def _get_root_data_from_netbox(self, nested: bool = False, **kwargs) -> LDAny:
         """Retrieve data from the Netbox.
 
         Request data based on the kwargs filter parameters and
         save the received objects to the NbForager.root.
+        Set extra `_nested` value in Netbox object.
+
+        :param bool nested: `True` - Request base and nested objects,
+            `False` - Request only base objects. Default id `False`
 
-        :param kwargs: Filter parameters.
+        :param kwargs: Filtering parameters.
 
         :return: List of Netbox objects. Update NbForager.root object.
         """
         nb_objects: LDAny = self.connector.get(**kwargs)
-        nb_objects = self._validate_ids(nb_objects)
         for nb_object in nb_objects:
-            self.root_d[nb_object["id"]] = nb_object
+            nb_object["_nested"] = nested
+            idx = int(nb_object["id"])
+            self.root_d[idx] = nb_object
         return nb_objects
 
-    @staticmethod
-    def _validate_ids(nb_objects: LDAny) -> LDAny:
-        """Check the IDs in the items. The ID should be a unique integer.
-
-        :param nb_objects: List of dictionaries containing Netbox objects.
-
-        :return: None. Logging an error if the ID does not match the conditions.
-        """
-        nb_objects_: LDAny = []
-        for nb_object in nb_objects:
-            id_ = nb_object.get("id")
-            if not isinstance(id_, int):
-                msg = f"TypeError: {id_=} {int} expected in {nb_object=}."
-                logging.error(msg)
-                continue
-            if id_ in nb_objects:
-                msg = f"ValueError: Duplicate {id_=} in nb_objects_ and {nb_object=}."
-                logging.error(msg)
-                continue
-            nb_objects_.append(nb_object)
-        return nb_objects_
-
     def _collect_nested_urls(self, nb_objects: LDAny) -> LStr:
         """Collect nested urls.
 
@@ -217,10 +222,20 @@ class Forager:
         :return: Nested URLs.
         """
         urls: LStr = h.nested_urls(nb_objects)
-        urls = missed_urls(urls=urls, tree=self.root)
+        urls = nb_tree.missed_urls(urls=urls, tree=self.root)
         urls = h.join_urls(urls)
-        urls = [s for s in urls if h.split_url(s)[0]]
-        return urls
+        urls = [s for s in urls if h.url_to_ami_items(s)[0]]
+
+        urls_: LStr = []
+
+        for url in urls:
+            app, model, _ = h.url_to_ami_items(url)
+            path = f"{app}/{model}/"
+            connector = self._get_connector(path)
+            urls_sliced = h.slice_url(url, max_len=connector.url_length)
+            urls_.extend(urls_sliced)
+
+        return urls_
 
     # noinspection PyProtectedMember
     def _get_path_params(self, urls: LStr) -> LT2StrDAny:
@@ -232,7 +247,7 @@ class Forager:
         """
         path_params: LT2StrDAny = []
         for url in urls:
-            app, model, _ = h.split_url(url)
+            app, model, _ = h.url_to_ami_items(url)
             path = f"{app}/{model}/"
             connector = self._get_connector(path)
             params_d = parse_qs(urlparse(url).query)
@@ -310,13 +325,20 @@ class Forager:
         connector = getattr(getattr(self.api, app), model)
         return connector
 
-    def _save_results(self, results):
-        # save
+    def _save_results(self, results: LDAny) -> None:
+        """Save Netbox objects to root NbTree object.
+
+        :param results: Data to be saved.
+        :return: None. root NbTree object.
+        """
         for data in results:
-            app, model, digit = h.split_url(data["url"])
+            app, model, idx_ = h.url_to_ami_items(data["url"])
+            idx = int(idx_)
             path = f"{app}/{model}"
             model_d: DiDAny = self._get_root_data(path)
-            model_d[int(digit)] = data
+            if idx not in model_d:
+                data["_nested"] = False
+                model_d[idx] = data
 
     def _get_pynb_data(self, path: str) -> DiAny:
         """Get data in self pynb by app/model path.
@@ -339,37 +361,3 @@ class Forager:
         app, model = h.path_to_attrs(path)
         data = getattr(getattr(self.root, app), model)
         return data
-
-
-def _find(objects: LDAny, **kwargs) -> LDAny:
-    """Find Netbox objects in tree by extended finding parameters.
-
-    :param objects: Netbox objects where searching is required using kwargs.
-    :param kwargs: Extended filtering parameters.
-    :return: Filtered Netbox objects.
-    """
-    if not kwargs:
-        return objects
-    (key, values), *key_values = list(kwargs.items())
-    if not isinstance(values, TLists):
-        values = [values]
-
-    objects_: LDAny = []
-    for data in objects:
-        keys = key.split("__")
-        if len(keys) <= 1:
-            keys = [key]
-        if keys[0] == "tags":
-            if len(keys) != 2:
-                raise ValueError(f"{keys=} {len(keys)=} expected 2.")
-            values_ = [d[keys[1]] for d in data["tags"]]
-            if vlist.is_in(values_, values):
-                objects_.append(data)
-        else:
-            value_ = NbParser(data).any(*keys)
-            if value_ in values:
-                objects_.append(data)
-
-    if key_values:
-        objects_ = _find(objects=objects_, **dict(key_values))
-    return objects_