peeringdb 2.4.1__py3-none-any.whl

peeringdb/__init__.py

@@ -0,0 +1,117 @@
+"""
+PeeringDB API
+"""
+
+import logging
+import sys
+from importlib import import_module
+from importlib import metadata as importlib_metadata
+from typing import TYPE_CHECKING, Optional, Union
+
+if TYPE_CHECKING:
+    from peeringdb.backend import Interface
+
+from peeringdb.util import get_log_level, str_to_bool
+
+__version__: str = importlib_metadata.version("peeringdb")
+_log_level: int = logging.INFO
+
+
+def _config_logs(
+    level: Optional[Union[str, int]] = None,
+    name: Optional[str] = None,
+    allow_other_loggers: Union[bool, str] = False,
+) -> None:
+    """
+    Set up or change logging configuration.
+
+    _config_logs() => idempotent setup;
+    _config_logs(L) => change log level
+    """
+    # print('_config_log', 'from %s' %name if name else '')
+    logging_format = "%(message)s"
+    # maybe better for log files
+    # FORMAT='[%(levelname)s]:%(message)s',
+
+    if isinstance(level, str):
+        level = get_log_level(level)
+
+    global _log_level
+    if level:
+        _log_level = level
+
+    if not isinstance(allow_other_loggers, bool):
+        try:
+            allow_other_loggers = str_to_bool(str(allow_other_loggers))
+        except Exception:
+            allow_other_loggers = False
+
+    if not allow_other_loggers:
+        # Reset handlers
+        for h in list(logging.root.handlers):
+            logging.root.removeHandler(h)
+
+    logging.basicConfig(level=_log_level, format=logging_format, stream=sys.stdout)
+    _log = logging.getLogger(__name__)
+    _log.setLevel(_log_level)
+
+    # external
+    for log in ["urllib3", "asyncio"]:
+        logging.getLogger(log).setLevel(_log_level)
+
+
+class BackendError(Exception):
+    pass
+
+
+# Map external module names to adaptor modules
+SUPPORTED_BACKENDS: dict[str, str] = {
+    "django_peeringdb": "django_peeringdb.client_adaptor",
+}
+
+__backend: Optional[tuple["Interface", tuple[str, str]]] = None
+
+
+def backend_initialized() -> bool:
+    return __backend is not None
+
+
+def _get() -> tuple["Interface", tuple[str, str]]:
+    global __backend
+    if __backend:
+        return __backend  # type: ignore[unreachable]
+    # mypy has trouble with global analysis here
+    raise BackendError("Backend not initialized")
+
+
+def get_backend() -> "Interface":
+    return _get()[0]
+
+
+def get_backend_info() -> tuple[str, str]:
+    return _get()[1]
+
+
+def initialize_backend(name: str, **kwargs: Union[str, int, bool, dict]) -> None:
+    global __backend
+    if __backend:
+        raise BackendError("Backend already initialized")
+
+    try:
+        modname = SUPPORTED_BACKENDS[name]
+    except KeyError:
+        raise ValueError(f"Not a supported backend module: '{name}'")
+    # Load internal module associated with the ORM module
+    supportmod = import_module(modname)
+    # Backend is any object returned from load_backend
+    backend = supportmod.load_backend(**kwargs)
+
+    backend.Backend.setup()
+    __backend = (backend.Backend(), (name, backend.__version__))
+
+
+# TODO
+# def is_valid_backend(backend): ...
+
+# namespace imports - import client at the end to avoid circular imports
+from peeringdb import client  # noqa: E402

peeringdb/_debug/__init__.py

@@ -0,0 +1,22 @@
+import logging
+
+# import pdb
+
+
+def log_validation_errors(backend, e, obj, k):
+    log = logging.getLogger("peeringdb.sync")
+    log.debug(f"{e} : errors: {e.message_dict}")
+    for k, v in e.message_dict.items():
+        field = backend.get_field(obj, k)
+        try:
+            log.debug(f"{k}: {getattr(obj, k)}, dict: {field.__dict__}")
+        except backend.object_missing_error():
+            log.debug(f"{k}: Missing Object, dict: {field.__dict__}")
+
+
+def try_or_debug(f):
+    try:
+        return f()
+    except Exception:
+        # pdb.set_trace() ???
+        raise

peeringdb/_debug/http.py

@@ -0,0 +1,8 @@
+import http.client
+import logging
+
+http.client.HTTPConnection.debuglevel = 1
+
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.DEBUG)
+requests_log.propagate = True

peeringdb/_sync.py

@@ -0,0 +1,111 @@
+"""
+Sync implementation module
+"""
+
+from collections import defaultdict
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from peeringdb.backend import Interface
+
+from peeringdb.util import group_fields
+
+
+def _field_resource(backend: "Interface", concrete: type, field: object) -> type:
+    field_name = getattr(field, "name", None)
+    if field_name is None:
+        raise ValueError("Field has no name")
+    field_concrete = backend.get_field_concrete(concrete, field_name)
+    if not isinstance(field_concrete, type):
+        raise ValueError(f"Expected type, got {type(field_concrete)}")
+    return backend.get_resource(field_concrete)
+
+
+def _get_subrow(
+    row: dict[str, Union[str, int, bool, list, dict]], fname: str, field: object
+) -> tuple[str, Union[str, int, bool, list, dict, None]]:
+    key = getattr(field, "column", None)
+    if key is not None and isinstance(key, str):
+        subrow = row.get(key)
+    else:
+        key = fname
+        subrow = None
+    if subrow is None:  # e.g. use "org" if "org_id" is missing
+        key = fname
+        try:
+            subrow = row[key]
+        except KeyError:
+            subrow = None
+    return key, subrow
+
+
+def extract_relations(
+    backend: "Interface", res: type, row: dict[str, Union[str, int, bool, list, dict]]
+) -> tuple[
+    dict[type, dict[Union[str, int], dict[str, Union[str, int, bool, list, dict]]]],
+    dict[type, set[Union[str, int]]],
+]:
+    field_groups = group_fields(backend, backend.get_concrete(res))
+    # Already-fetched, and id-only refs
+    fetched: dict = defaultdict(dict)
+    dangling = defaultdict(set)
+
+    # Handle subrows that might be shallow (id) or deep (dict)
+    def _handle_subrow(resource, subrow):
+        if isinstance(subrow, dict):
+            pk = subrow["id"]
+            fetched[resource][pk] = subrow
+        elif subrow is None:
+            return
+        else:
+            pk = subrow
+            dangling[resource].add(pk)
+        return pk
+
+    for fname, field in field_groups["single_refs"].items():
+        fieldres = _field_resource(backend, backend.get_concrete(res), field)
+        _, subrow = _get_subrow(row, fname, field)
+        _handle_subrow(fieldres, subrow)
+
+    for fname, field in field_groups["many_refs"].items():
+        fieldres = _field_resource(backend, backend.get_concrete(res), field)
+        many_data = row.get(fname, [])
+        if isinstance(many_data, list):
+            for subrow in many_data:
+                _handle_subrow(fieldres, subrow)
+
+    return fetched, dangling
+
+
+def set_single_relations(
+    backend: "Interface",
+    res: type,
+    obj: object,
+    row: dict[str, Union[str, int, bool, list, dict]],
+) -> None:
+    field_groups = group_fields(backend, backend.get_concrete(res))
+    for fname, field in field_groups["single_refs"].items():
+        key, subrow = _get_subrow(row, fname, field)
+        if isinstance(subrow, dict):
+            pk = subrow["id"]
+        else:
+            pk = subrow
+        setattr(obj, key, pk)
+
+
+def set_many_relations(
+    backend: "Interface",
+    res: type,
+    obj: object,
+    row: dict[str, Union[str, int, bool, list, dict]],
+) -> None:
+    field_groups = group_fields(backend, backend.get_concrete(res))
+    for fname, field in field_groups["many_refs"].items():
+        fieldres = _field_resource(backend, backend.get_concrete(res), field)
+        pks_data = row.get(fname, [])
+        if isinstance(pks_data, list):
+            pks = pks_data
+        else:
+            pks = []
+        objs = [backend.get_object(backend.get_concrete(fieldres), pk) for pk in pks]
+        backend.set_relation_many_to_many(obj, fname, objs)

peeringdb/_update.py

@@ -0,0 +1,309 @@
+"""
+Module defining main interface classes for sync
+"""
+
+import logging
+from datetime import datetime
+from typing import TYPE_CHECKING, Optional, Union
+
+if TYPE_CHECKING:
+    pass
+
+from peeringdb import config, get_backend
+from peeringdb._sync import extract_relations, set_many_relations, set_single_relations
+from peeringdb.fetch import Fetcher
+from peeringdb.private import private_data_has_been_fetched
+from peeringdb.util import group_fields, log_error
+
+
+class Updater:
+    """
+    Handles initial and incremental update from a PeeringDB remote API
+    to the local backend.
+
+    The updater is responsible for creating and updating objects in the local
+    backend. It does this by fetching objects from the remote API and creating
+    or updating the corresponding objects in the local backend.
+    """
+
+    def __init__(self, fetcher: Fetcher):
+        self._log = logging.getLogger(__name__)
+        self.resources: dict = {}
+        self.backend = get_backend()
+        self.fetcher = fetcher
+        self.config = config.load_config()
+
+    def copy_object(self, new):
+        """
+        Copies a new object to an existing one
+        :return:
+        """
+        old = self.backend.get_object(new.__class__, new.id)
+        for field in self.backend.get_fields(new.__class__):
+            try:
+                setattr(old, field.name, getattr(new, field.name))
+            except TypeError as e:
+                if "Direct assignment" in str(e):
+                    pass  # Ignore refs
+                else:
+                    raise
+
+        self.clean_obj(old)
+        self.backend.save(old)
+
+    def clean_obj(self, obj):
+        """
+        Run object through backend validation
+
+        Will raise an exception if the object is not valid
+        """
+
+        try:
+            self.backend.clean(obj)
+        except self.backend.validation_error() as e:
+            # e.message_dict contains field names as keys and lists of errors as values
+            for errors in e.message_dict.values():
+                for error in errors:
+                    # Checking if the error message is the one we want to ignore
+                    # field is allowed to be None in the db, but not blank according
+                    # to backend validation. We ignore this error since the data is
+                    # already validated and writing None to the db is fine.
+                    if error != "This field cannot be blank.":
+                        raise e
+
+    def create_obj(
+        self, row: dict[str, Union[str, int, bool, list, dict]], res: type
+    ) -> tuple[object, bool]:  # noqa: C901
+        """
+        Create a model instance from a row
+        :param row: Object from API
+        :param res: Resource to create
+        """
+        _, dangling = extract_relations(self.backend, res, row)
+        for resource, pks in dangling.items():
+            for pk in pks:
+                # Check if we have it
+                rel_obj = None
+                try:
+                    self.backend.get_object(self.backend.get_concrete(resource), pk)
+                except self.backend.object_missing_error(
+                    self.backend.get_concrete(resource)
+                ):
+                    # We dont have the dangling relationship, so we try to fetch it
+                    # from the api and create it.
+
+                    self._log.info("Fetching dangling relationship %s %s", resource, pk)
+                    related_row = self.fetcher.get(resource.tag, int(pk))
+
+                    # instantiate the relationship object
+
+                    rel_obj, _ = self.create_obj(related_row, resource)
+                    try:
+                        self.clean_obj(rel_obj)
+                    except self.backend.validation_error() as e:
+                        self._log.error(
+                            "Failed to clean dangling object %s %s: %s", resource, pk, e
+                        )
+                        return None, False
+
+                    # save the relationship object
+
+                    if rel_obj:
+                        self.backend.save(rel_obj)
+
+        # Initialize object
+        field_groups = group_fields(self.backend, self.backend.get_concrete(res))
+        try:
+            obj = self.backend.get_object(self.backend.get_concrete(res), row["id"])
+        except self.backend.object_missing_error(self.backend.get_concrete(res)):
+            tbl = self.backend.get_concrete(res)
+            obj = tbl()
+
+        # set_scalars
+        for fname, field in field_groups["scalars"].items():
+            value = row.get(fname, getattr(obj, fname, None))
+            value = self.backend.convert_field(obj.__class__, fname, value)
+
+            # TODO: datetimes are strings for some reason
+            if (
+                fname in ["created", "updated", "rir_status_updated", "ixf_last_import"]
+                and isinstance(value, str)
+            ) or (getattr(obj, "tzinfo", None) is not None):
+                value = datetime.fromisoformat(value.rstrip("Z"))
+            # elif (isinstance(value, str) and "T" in value and
+            #       "Z" in value and "-" in value):
+            #     print("NOT DATETIME", fname, value, type(value))
+
+            # Remove timezone info
+            if isinstance(value, datetime):
+                value = value.replace(tzinfo=None)
+
+            setattr(obj, fname, value)
+            self._log.debug("  %s: %s (%s)", fname, value, type(value))
+
+        set_single_relations(self.backend, res, obj, row)
+        set_many_relations(self.backend, res, obj, row)
+
+        try:
+            self.clean_obj(obj)
+        except self.backend.validation_error() as e:
+            self._log.debug("[%s] Failed to clean %s %s: %s", res.tag, res, obj, e)
+            if "already exists" in str(e):
+                self.update_collision(res, row, e)
+                self.clean_obj(obj)
+            return obj, True
+
+        return obj, False
+
+    def _handle_initial_sync(self, entries: list, res):
+        """
+        Called during the first sync of a resource
+
+        This will do a batch create of all objects in the resource
+        :param entries: List of objects from API
+        :param res: Resource to sync
+        """
+
+        objs = []
+        for row in entries:
+            try:
+                obj, _ = self.create_obj(row, res)
+                objs.append(obj)
+            except self.backend.object_missing_error(self.backend.get_concrete(res)):
+                try:
+                    obj, _ = self.create_obj(row, res)
+                    self.backend.save(obj)
+                except Exception as e:
+                    obj_id = row.get("id", "Unknown")
+                    self._log.info(f"Error creating {res.tag} with id {obj_id}: {e}")
+                    log_error(self.config, res.tag, row.get("id", "Unknown"), str(e))
+            except Exception as e:
+                obj_id = row.get("id", "Unknown")
+                self._log.info(f"Error updating {res.tag} with id {obj_id}: {e}")
+                log_error(self.config, res.tag, row.get("id", "Unknown"), str(e))
+
+        self.backend.get_concrete(res).objects.bulk_create(objs)
+
+    def _handle_incremental_sync(self, entries: list, res):
+        """
+        Called during an incremental sync of a resource (i.e. not the first sync)
+
+        Entries will only contain objects that have changed since the last sync
+        :param entries: List of objects from API
+        :param res: Resource to sync
+        """
+
+        for row in entries:
+            try:
+                self.backend.get_object(self.backend.get_concrete(res), row["id"])
+                obj, _ = self.create_obj(row, res)
+                self.copy_object(obj)
+            except self.backend.object_missing_error(self.backend.get_concrete(res)):
+                try:
+                    obj, _ = self.create_obj(row, res)
+                    self.backend.save(obj)
+                except Exception as e:
+                    obj_id = row.get("id", "Unknown")
+                    self._log.info(f"Error creating {res.tag} with id {obj_id}: {e}")
+                    log_error(self.config, res.tag, row.get("id", "Unknown"), str(e))
+            except Exception as e:
+                obj_id = row.get("id", "Unknown")
+                self._log.info(f"Error updating {res.tag} with id {obj_id}: {e}")
+                log_error(self.config, res.tag, row.get("id", "Unknown"), str(e))
+
+    def update_all(
+        self,
+        rs: list[type],
+        since: Optional[int] = None,
+        skip: Union[list[str], None] = None,
+        fetch_private: bool = False,
+    ):
+        """
+        Update all objects of a given type
+        :param rs: List of resources to update
+        :param since: Unix timestamp of last update
+        :param skip: List of resource tags to skip
+        """
+
+        for res in rs:
+            if skip is not None:
+                for i in skip:
+                    self.fetcher.load(i, since)
+
+            if skip and res.tag in skip:
+                self._log.info("[%s] Skipping", res.tag)
+                continue
+
+            if since is None:
+                _since = self.backend.last_change(self.backend.get_concrete(res))
+            else:
+                _since = since
+
+            initial_private = False
+            if fetch_private:
+                initial_private = not private_data_has_been_fetched(self.backend, res)
+
+            self.fetcher.load(
+                res.tag,
+                _since + 1 if _since and isinstance(_since, int) else None,
+                fetch_private=fetch_private,
+                initial_private=initial_private,
+            )
+            entries = self.fetcher.entries(res.tag)
+            self._log.info("[%s] Processing %d objects", res.tag, len(entries))
+
+            if not _since:
+                self._handle_initial_sync(entries, res)
+            else:
+                self._handle_incremental_sync(entries, res)
+
+    def update_one(self, res, pk: int, depth=0):
+        """
+        Update a single object
+        :param res: Resource to update
+        :param pk: Primary key of object to update
+        :param depth: Depth of recursion
+        :return:
+        """
+        if depth != 0:
+            # no longer relevant, deprecation warning
+            self._log.warning(
+                "update_one: depth parameter is not used and will be removed "
+                "in a future version"
+            )
+
+        row = self.fetcher.get(res.tag, pk, depth=0, force_fetch=True)
+        # create object instance (unsaved)
+        obj, _ = self.create_obj(row, res)
+        try:
+            # attempt update existing instance of object (if exists, will save)
+            self.copy_object(obj)
+        except self.backend.object_missing_error(self.backend.get_concrete(res)):
+            # object does not exist, create object instance and save as
+            # new object
+            obj, _ = self.create_obj(row, res)
+            self.backend.save(obj)
+
+    def update_collision(self, res, row: dict, exc: Exception):
+        """
+        Sometimes we encounter edge-case collisions triggered by a
+        unique constraint validation error. This function attempts to
+        resolve the collision by updating the colliding object from the api
+        :param res: Resource to update
+        :param row: Row from API
+        :param exc: Exception raised by validation error
+        """
+
+        filters = {}
+        for field, error_message in exc.error_dict.items():
+            if "already exists" in str(error_message):
+                filters[field] = row[field]
+
+        for field, value in filters.items():
+            self._log.debug(
+                "[%s] Updating collision %s %s=%s", res.tag, res, field, value
+            )
+            collision = self.backend.get_object_by(
+                self.backend.get_concrete(res), field, value
+            )
+            self.update_one(res, collision.id)