elody 0.0.63__py3-none-any.whl → 0.0.163__py3-none-any.whl

elody/job.py

@@ -0,0 +1,95 @@
+from elody.object_configurations.job_configuration import JobConfiguration
+
+
+_config = JobConfiguration()
+_create = _config.crud()["creator"]
+_post_crud_hook = _config.crud()["post_crud_hook"]
+
+
+def start_job(
+    name,
+    job_type,
+    *,
+    get_rabbit,
+    get_user_context=None,
+    user_email=None,
+    parent_id=None,
+    id_of_document_job_was_initiated_for=None,
+    type_of_document_job_was_initiated_for=None,
+) -> str:
+    relations = []
+    if parent_id:
+        relations.append({"key": parent_id, "type": "hasParentJob"})
+    if id_of_document_job_was_initiated_for and type_of_document_job_was_initiated_for:
+        relations.append(
+            {"key": id_of_document_job_was_initiated_for, "type": "isJobOf"}
+        )
+
+    job = _create(
+        {
+            "metadata": [
+                {"key": "name", "value": name},
+                {"key": "status", "value": "running"},
+                {"key": "type", "value": job_type},
+            ],
+            "relations": relations,
+            "type": "job",
+        },
+        get_user_context=get_user_context
+        or (lambda: type("UserContext", (object,), {"email": user_email})()),
+    )
+    del job["computed_values"]["created_at"]
+
+    _post_crud_hook(
+        crud="create", document=job, parent_id=parent_id, get_rabbit=get_rabbit
+    )
+    __patch_document_job_was_initiated_for(
+        id_of_document_job_was_initiated_for,
+        type_of_document_job_was_initiated_for,
+        get_rabbit,
+    )
+    return job["_id"]
+
+
+def finish_job(
+    id,
+    id_of_document_job_was_initiated_for=None,
+    type_of_document_job_was_initiated_for=None,
+    *,
+    get_rabbit,
+):
+    document = {
+        "id": id,
+        "patch": {
+            "metadata": [{"key": "status", "value": "finished"}],
+            "relations": ([] if id_of_document_job_was_initiated_for else []),
+        },
+    }
+    _post_crud_hook(crud="update", document=document, get_rabbit=get_rabbit)
+    __patch_document_job_was_initiated_for(
+        id_of_document_job_was_initiated_for,
+        type_of_document_job_was_initiated_for,
+        get_rabbit,
+    )
+
+
+def fail_job(id, exception_message, *, get_rabbit):
+    document = {
+        "id": id,
+        "patch": {
+            "metadata": [
+                {"key": "info", "value": exception_message},
+                {"key": "status", "value": "failed"},
+            ]
+        },
+    }
+    _post_crud_hook(crud="update", document=document, get_rabbit=get_rabbit)
+
+
+def __patch_document_job_was_initiated_for(id, type, get_rabbit):
+    if id and type:
+        document = {
+            "document_info_job_was_initiated_for": {"id": id, "type": type},
+            "patch": {"relations": [{"key": id, "type": "hasJob"}]},
+        }
+        _post_crud_hook(crud="update", document=document, get_rabbit=get_rabbit)

elody/loader.py

@@ -2,7 +2,7 @@ import elody.util as util
 import json
 import os
 
-from elody.policies.permission_handler import set_permissions
+from apscheduler.triggers.cron import CronTrigger
 from importlib import import_module
 from inuits_policy_based_auth.exceptions import (
     PolicyFactoryException,
@@ -17,9 +17,32 @@ def load_apps(flask_app, logger):
             flask_app.register_blueprint(api_bp)
 
 
-def load_policies(policy_factory, logger, permissions={}):
+def load_jobs(scheduler, logger):
+    apps = util.read_json_as_dict(os.getenv("APPS_MANIFEST"), logger)
+    for app in apps:
+        for job, job_properties in apps[app].get("jobs", {}).items():
+            try:
+                job_class = __get_class_from_module(
+                    import_module(f"apps.{app}.cron_jobs.{job}")
+                )
+                scheduler.add_job(
+                    job_class(),
+                    CronTrigger.from_crontab(
+                        job_properties.get("expression", "0 0 * * *")
+                    ),
+                )
+            except ModuleNotFoundError:
+                pass
+
+
+def load_policies(
+    policy_factory, logger, permissions: dict = {}, placeholders: list[str] = []
+):
     if permissions:
-        set_permissions(permissions)
+        from elody.policies.permission_handler import set_permissions
+
+        set_permissions(permissions, placeholders)
+
     apps = util.read_json_as_dict(os.getenv("APPS_MANIFEST", ""), logger)
     for app in apps:
         try:
@@ -43,6 +66,7 @@ def load_policies(policy_factory, logger, permissions={}):
 
 
 def load_queues(logger):
+    import_module("resources.queues")
     apps = util.read_json_as_dict(os.getenv("APPS_MANIFEST"), logger)
     for app in apps:
         try:
@@ -66,11 +90,15 @@ def __get_class(app, auth_type, policy_module_name):
             pass
     else:
         raise ModuleNotFoundError(f"Policy {policy_module_name} not found")
-    policy_class_name = module.__name__.split(".")[-1].title().replace("_", "")
-    policy = getattr(module, policy_class_name)
+    policy = __get_class_from_module(module)
     return policy
 
 
+def __get_class_from_module(module):
+    class_name = module.__name__.split(".")[-1].title().replace("_", "")
+    return getattr(module, class_name)
+
+
 def __instantiate_authentication_policy(policy_module_name, policy, logger):
     allow_anonymous_users = os.getenv("ALLOW_ANONYMOUS_USERS", False) in [
         "True",

elody/migration/base_object_migrator.py

@@ -0,0 +1,18 @@
+class BaseObjectMigrator:
+    def __init__(self, *, status, silent=False):
+        self._status = status
+        self._silent = silent
+
+    @property
+    def status(self):
+        return self._status
+
+    @property
+    def silent(self):
+        return self._silent
+
+    def bulk_migrate(self, *, dry_run=False):  # pyright: ignore
+        pass
+
+    def lazy_migrate(self, item, *, dry_run=False):  # pyright: ignore
+        return item

elody/object_configurations/base_object_configuration.py

@@ -0,0 +1,174 @@
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from elody.migration.base_object_migrator import BaseObjectMigrator
+
+
+class BaseObjectConfiguration(ABC):
+    SCHEMA_TYPE = "elody"
+    SCHEMA_VERSION = 1
+
+    @abstractmethod
+    def crud(self):
+        return {
+            "creator": lambda post_body, **kwargs: post_body,
+            "document_content_patcher": lambda *, document, content, overwrite=False, **kwargs: self._document_content_patcher(
+                document=document,
+                content=content,
+                overwrite=overwrite,
+                **kwargs,
+            ),
+            "nested_matcher_builder": lambda object_lists, keys_info, value: self.__build_nested_matcher(
+                object_lists, keys_info, value
+            ),
+            "post_crud_hook": lambda **kwargs: None,
+            "pre_crud_hook": lambda *, document, **kwargs: document,
+            "storage_type": "db",
+        }
+
+    @abstractmethod
+    def document_info(self):
+        return {}
+
+    @abstractmethod
+    def logging(self, flat_document, **kwargs):
+        info_labels = {
+            "uuid": flat_document.get("_id"),
+            "type": flat_document.get("type"),
+            "schema": f"{flat_document.get('schema.type')}:{flat_document.get('schema.version')}",
+        }
+        try:
+            from policy_factory import get_user_context  # pyright: ignore
+
+            user_context = get_user_context()
+            info_labels["http_method"] = user_context.bag.get("http_method")
+            info_labels["requested_endpoint"] = user_context.bag.get(
+                "requested_endpoint"
+            )
+            info_labels["full_path"] = user_context.bag.get("full_path")
+            info_labels["preferred_username"] = user_context.preferred_username
+            info_labels["email"] = user_context.email
+            info_labels["user_roles"] = ", ".join(user_context.x_tenant.roles)
+            info_labels["x_tenant"] = user_context.x_tenant.id
+        except Exception:
+            pass
+        return {"info_labels": info_labels, "loki_indexed_info_labels": {}}
+
+    @abstractmethod
+    def migration(self):
+        return BaseObjectMigrator(status="disabled")
+
+    @abstractmethod
+    def serialization(self, from_format, to_format):
+        def serializer(document, **_):
+            return document
+
+        return serializer
+
+    @abstractmethod
+    def validation(self):
+        def validator(http_method, content, item, **_):
+            pass
+
+        return "function", validator
+
+    def _document_content_patcher(
+        self, *, document, content, overwrite=False, **kwargs
+    ):
+        raise NotImplementedError(
+            "Provide concrete implementation in child object configuration"
+        )
+
+    def _merge_object_lists(self, source, target, object_list_key):
+        for target_item in target:
+            for source_item in source:
+                if source_item[object_list_key] == target_item[object_list_key]:
+                    source.remove(source_item)
+        return [*source, *target]
+
+    def _get_user_context_id(self):
+        try:
+            from policy_factory import get_user_context  # pyright: ignore
+
+            return get_user_context().id
+        except Exception:
+            return None
+
+    def _sanitize_document(self, *, document, **kwargs):
+        sanitized_document = {}
+        document_deepcopy = deepcopy(document)
+        for key, value in document_deepcopy.items():
+            if isinstance(value, dict):
+                sanitized_value = BaseObjectConfiguration._sanitize_document(
+                    self, document=value
+                )
+                if sanitized_value:
+                    sanitized_document[key] = sanitized_value
+            elif value:
+                sanitized_document[key] = value
+        return sanitized_document
+
+    def _should_create_history_object(self):
+        try:
+            from policy_factory import get_user_context  # pyright: ignore
+
+            get_user_context()
+            return bool(self.crud().get("collection_history"))
+        except Exception:
+            return False
+
+    def _sort_document_keys(self, document):
+        def sort_keys(data):
+            if isinstance(data, dict):
+                sorted_items = {key: data.pop(key) for key in sorted(data.keys())}
+                for key, value in sorted_items.items():
+                    data[key] = sort_keys(value)
+                return data
+            elif isinstance(data, list):
+                if all(isinstance(i, str) for i in data):
+                    data.sort()
+                    return data
+                else:
+                    for index, item in enumerate(data):
+                        data[index] = sort_keys(item)
+                    return data
+            else:
+                return data
+
+        for key, value in self.document_info().get("object_lists", {}).items():
+            if document.get(key):
+                document[key] = sorted(
+                    document[key], key=lambda property: property[value]
+                )
+        sort_keys(document)
+        return document
+
+    def __build_nested_matcher(self, object_lists, keys_info, value, index=0):
+        if index == 0 and not any(info["object_list"] for info in keys_info):
+            if value in ["ANY_MATCH", "NONE_MATCH"]:
+                value = {"$exists": value == "ANY_MATCH"}
+            return {".".join(info["key"] for info in keys_info): value}
+
+        info = keys_info[index]
+
+        if info["object_list"]:
+            elem_match = {
+                "$elemMatch": {
+                    object_lists[info["object_list"]]: info["object_key"],
+                    **self.__build_nested_matcher(
+                        object_lists, keys_info[index + 1 :], value, 0
+                    ),
+                }
+            }
+            if value in ["ANY_MATCH", "NONE_MATCH"]:
+                elem_match_with_exists_operator = deepcopy(elem_match)
+                del elem_match["$elemMatch"][keys_info[index + 1]["key"]]
+                if value == "NONE_MATCH":
+                    return {
+                        "NOR_MATCHER": [
+                            {info["key"]: {"$all": [elem_match]}},
+                            {info["key"]: {"$all": [elem_match_with_exists_operator]}},
+                        ]
+                    }
+            return elem_match if index > 0 else {info["key"]: {"$all": [elem_match]}}
+
+        raise Exception(f"Unable to build nested matcher. See keys_info: {keys_info}")

elody/object_configurations/elody_configuration.py

@@ -0,0 +1,144 @@
+from datetime import datetime, timezone
+from elody.object_configurations.base_object_configuration import (
+    BaseObjectConfiguration,
+)
+from elody.util import flatten_dict
+from uuid import uuid4
+
+
+class ElodyConfiguration(BaseObjectConfiguration):
+    SCHEMA_TYPE = "elody"
+    SCHEMA_VERSION = 1
+
+    def crud(self):
+        crud = {
+            "collection": "entities",
+            "collection_history": "history",
+            "creator": lambda post_body, **kwargs: self._creator(post_body, **kwargs),
+            "post_crud_hook": lambda **kwargs: self._post_crud_hook(**kwargs),
+            "pre_crud_hook": lambda **kwargs: self._pre_crud_hook(**kwargs),
+        }
+        return {**super().crud(), **crud}
+
+    def document_info(self):
+        return {"object_lists": {"metadata": "key", "relations": "type"}}
+
+    def logging(self, flat_document, **kwargs):
+        return super().logging(flat_document, **kwargs)
+
+    def migration(self):
+        return super().migration()
+
+    def serialization(self, from_format, to_format):
+        return super().serialization(from_format, to_format)
+
+    def validation(self):
+        return super().validation()
+
+    def _creator(
+        self,
+        post_body,
+        *,
+        flat_post_body={},
+        document_defaults={},
+    ):
+        if not flat_post_body:
+            flat_post_body = flatten_dict(
+                self.document_info()["object_lists"], post_body
+            )
+        _id = document_defaults.get("_id", str(uuid4()))
+
+        identifiers = []
+        for property in self.document_info().get("identifier_properties", []):
+            if identifier := flat_post_body.get(f"metadata.{property}.value"):
+                identifiers.append(identifier)
+
+        template = {
+            "_id": _id,
+            "computed_values": {
+                "created_at": datetime.now(timezone.utc),
+                "event": "create",
+            },
+            "identifiers": list(
+                set([_id, *identifiers, *document_defaults.pop("identifiers", [])])
+            ),
+            "metadata": [],
+            "relations": [],
+            "schema": {"type": self.SCHEMA_TYPE, "version": self.SCHEMA_VERSION},
+        }
+        if user_context_id := self._get_user_context_id():
+            template["computed_values"]["created_by"] = user_context_id
+
+        for key, object_list_key in self.document_info()["object_lists"].items():
+            if not key.startswith("lookup.virtual_relations"):
+                post_body[key] = self._merge_object_lists(
+                    document_defaults.get(key, []),
+                    post_body.get(key, []),
+                    object_list_key,
+                )
+        document = {**template, **document_defaults, **post_body}
+
+        document = self._sanitize_document(
+            document=document,
+            object_list_name="metadata",
+            object_list_value_field_name="value",
+        )
+        document = self._sort_document_keys(document)
+        return document
+
+    def _document_content_patcher(
+        self, *, document, content, overwrite=False, **kwargs
+    ):
+        object_lists = self.document_info().get("object_lists", {})
+        if overwrite:
+            document = content
+        else:
+            for key, value in content.items():
+                if key in object_lists:
+                    if key != "relations":
+                        for value_element in value:
+                            for item_element in document[key]:
+                                if (
+                                    item_element[object_lists[key]]
+                                    == value_element[object_lists[key]]
+                                ):
+                                    document[key].remove(item_element)
+                                    break
+                    document[key].extend(value)
+                else:
+                    document[key] = value
+
+        return document
+
+    def _post_crud_hook(self, **kwargs):
+        pass
+
+    def _pre_crud_hook(self, *, crud, document={}, **kwargs):
+        if document:
+            document = self._sanitize_document(
+                document=document,
+                object_list_name="metadata",
+                object_list_value_field_name="value",
+            )
+            document = self.__patch_document_computed_values(crud, document)
+            document = self._sort_document_keys(document)
+        return document
+
+    def _sanitize_document(
+        self, *, document, object_list_name, object_list_value_field_name, **kwargs
+    ):
+        sanitized_document = super()._sanitize_document(document=document)
+        object_list = document[object_list_name]
+        for element in object_list:
+            if not element[object_list_value_field_name]:
+                sanitized_document[object_list_name].remove(element)
+        return sanitized_document
+
+    def __patch_document_computed_values(self, crud, document):
+        if not document.get("computed_values"):
+            document["computed_values"] = {}
+        document["computed_values"].update({"event": crud})
+        document["computed_values"].update({"modified_at": datetime.now(timezone.utc)})
+        if email := self._get_user_context_id():
+            document["computed_values"].update({"modified_by": email})
+        return document

elody/object_configurations/job_configuration.py

@@ -0,0 +1,65 @@
+from elody.object_configurations.elody_configuration import (
+    ElodyConfiguration,
+)
+from elody.util import send_cloudevent
+from os import getenv
+
+
+class JobConfiguration(ElodyConfiguration):
+    SCHEMA_TYPE = "elody"
+    SCHEMA_VERSION = 1
+
+    def crud(self):
+        crud = {"collection": "jobs", "collection_history": ""}
+        return {**super().crud(), **crud}
+
+    def document_info(self):
+        return super().document_info()
+
+    def logging(self, flat_document, **kwargs):
+        return super().logging(flat_document, **kwargs)
+
+    def migration(self):
+        return super().migration()
+
+    def serialization(self, from_format, to_format):
+        return super().serialization(from_format, to_format)
+
+    def validation(self):
+        return super().validation()
+
+    def _creator(self, post_body, *, get_user_context={}, **_):
+        document = super()._creator(post_body)
+        if email := get_user_context().email:
+            document["computed_values"]["created_by"] = email
+        return document
+
+    def _post_crud_hook(self, *, crud, document, get_rabbit, **kwargs):
+        if crud == "create":
+            send_cloudevent(
+                get_rabbit(),
+                getenv("MQ_EXCHANGE", "dams"),
+                "dams.job_created",
+                document,
+            )
+            if parent_id := kwargs.get("parent_id"):
+                send_cloudevent(
+                    get_rabbit(),
+                    getenv("MQ_EXCHANGE", "dams"),
+                    "dams.job_changed",
+                    {
+                        "id": parent_id,
+                        "patch": {
+                            "relations": [
+                                {"key": document["_id"], "type": "isParentJobOf"}
+                            ]
+                        },
+                    },
+                )
+        elif crud == "update":
+            send_cloudevent(
+                get_rabbit(),
+                getenv("MQ_EXCHANGE", "dams"),
+                "dams.job_changed",
+                document,
+            )