dmart 1.4.40.post8__py3-none-any.whl

dmart/plugins/redis_db_update/plugin.py

@@ -0,0 +1,188 @@
+import sys
+from models.core import ActionType, Attachment, PluginBase, Event, Space
+from utils.helpers import camel_case
+from data_adapters.file.adapter_helpers import generate_payload_string
+from data_adapters.adapter import data_adapter as db
+from models import core
+from models.enums import ContentType, ResourceType
+from data_adapters.file.redis_services import RedisServices
+from fastapi.logger import logger
+from data_adapters.file.create_index import main as reload_redis
+from utils.settings import settings
+from typing import Any
+
+
+class Plugin(PluginBase):
+    async def hook(self, data: Event):
+        if settings.active_data_db == "sql":
+            return
+
+        self.data = data
+        # Type narrowing for PyRight
+        if (
+            not isinstance(data.shortname, str)
+            or not isinstance(data.resource_type, ResourceType)
+            or not isinstance(data.attributes, dict)
+        ):
+            logger.error("invalid data at redis_db_update")
+            return
+
+        spaces = await db.get_spaces()
+        if (
+            data.space_name not in spaces
+            or not Space.model_validate_json(spaces[data.space_name]).indexing_enabled
+        ):
+            return
+
+        class_type = getattr(
+            sys.modules["models.core"],
+            camel_case(core.ResourceType(data.resource_type)),
+        )
+        if issubclass(class_type, Attachment):
+            await self.update_parent_entry_payload_string()
+            return
+
+        async with RedisServices() as redis_services:
+            if data.resource_type == ResourceType.folder and data.action_type in [
+                ActionType.delete,
+                ActionType.move,
+            ]:
+                await reload_redis(for_space=data.space_name)
+                return
+
+            if data.action_type == ActionType.delete:
+                doc_id = redis_services.generate_doc_id(
+                    data.space_name,
+                    "meta",
+                    data.shortname,
+                    data.subpath,
+                )
+                meta_doc = await redis_services.get_doc_by_id(doc_id)
+                # Delete meta doc
+                await redis_services.delete_doc(
+                    data.space_name,
+                    "meta",
+                    data.shortname,
+                    data.subpath,
+                )
+                # Delete payload doc
+                await redis_services.delete_doc(
+                    data.space_name,
+                    (
+                        meta_doc.get("payload", {}).get("schema_shortname", "meta")
+                        if meta_doc
+                        else "meta"
+                    ),
+                    data.shortname,
+                    data.subpath,
+                )
+                return
+            try:
+                meta = await db.load(
+                    space_name=data.space_name,
+                    subpath=data.subpath,
+                    shortname=data.shortname,
+                    class_type=class_type,
+                    user_shortname=data.user_shortname,
+                )
+            except Exception as _:
+                return
+
+            if data.action_type in [
+                ActionType.create,
+                ActionType.update,
+                ActionType.progress_ticket,
+            ]:
+                meta_doc_id, meta_json = redis_services.prepare_meta_doc(
+                    data.space_name, data.subpath, meta
+                )
+                payload: dict[str, Any] = {}
+                if (
+                    meta.payload
+                    and meta.payload.content_type == ContentType.json
+                    and meta.payload.body is not None
+                ):
+                    mypayload = await db.load_resource_payload(
+                        space_name=data.space_name,
+                        subpath=data.subpath,
+                        filename=str(meta.payload.body),
+                        class_type=class_type,
+                    )
+                    payload = mypayload if mypayload else {}
+
+                meta_json["payload_string"] = await generate_payload_string(
+                    db,
+                    space_name=data.space_name,
+                    subpath=meta_json["subpath"],
+                    shortname=meta_json["shortname"],
+                    payload=payload,
+                ) if settings.store_payload_string else ""
+
+                await redis_services.save_doc(meta_doc_id, meta_json)
+                if meta.payload:
+                    payload.update(meta_json)
+                    await redis_services.save_payload_doc(
+                        data.space_name,
+                        data.subpath,
+                        meta,
+                        payload,
+                        data.resource_type,
+                    )
+
+            elif data.action_type == ActionType.move:
+                await redis_services.move_meta_doc(
+                    data.space_name,
+                    data.attributes["src_shortname"],
+                    data.attributes["src_subpath"],
+                    data.subpath,
+                    meta,
+                )
+                if meta.payload and meta.payload.schema_shortname:
+                    await redis_services.move_payload_doc(
+                        data.space_name,
+                        meta.payload.schema_shortname,
+                        data.attributes["src_shortname"],
+                        data.attributes["src_subpath"],
+                        meta.shortname,
+                        data.subpath,
+                    )
+
+    async def update_parent_entry_payload_string(self) -> None:
+        async with RedisServices() as redis_services:
+            # get the parent meta doc
+            subpath_parts = self.data.subpath.strip("/").split("/")
+            if len(subpath_parts) <= 1:
+                return
+            parent_subpath, parent_shortname = (
+                "/".join(subpath_parts[:-1]),
+                subpath_parts[-1],
+            )
+            doc_id = redis_services.generate_doc_id(
+                self.data.space_name,
+                "meta",
+                parent_shortname,
+                parent_subpath,
+            )
+            meta_doc: dict = await redis_services.get_doc_by_id(doc_id)
+
+            if meta_doc is None:
+                raise Exception("Meta doc not found")
+
+            payload = {}
+            if meta_doc.get("payload_doc_id"):
+                payload_doc = await redis_services.get_doc_by_id(
+                    meta_doc["payload_doc_id"]
+                )
+                payload = {k: v for k, v in payload_doc.items() if k not in meta_doc}
+
+            # generate the payload string
+            meta_doc["payload_string"] = await generate_payload_string(
+                db,
+                space_name=self.data.space_name,
+                subpath=parent_subpath,
+                shortname=parent_shortname,
+                payload=payload,
+            ) if settings.store_payload_string else ""
+
+            # update parent meta doc
+            await redis_services.save_doc(doc_id, meta_doc)

dmart/plugins/resource_folders_creation/config.json

@@ -0,0 +1,12 @@
+{
+  "shortname": "resource_folders_creation",
+	"is_active": true,
+	"filters": {
+		"subpaths": ["__ALL__"],
+		"resource_types": ["user", "space"],
+		"schema_shortnames": ["__ALL__"],
+		"actions": ["create"]
+	},
+	"type": "hook",
+	"listen_time": "after"
+}

dmart/plugins/resource_folders_creation/plugin.py

@@ -0,0 +1,81 @@
+from models.core import Folder, PluginBase, Event
+from models.enums import ResourceType
+from data_adapters.file.redis_services import RedisServices
+from fastapi.logger import logger
+from data_adapters.adapter import data_adapter as db
+from utils.settings import settings
+
+
+class Plugin(PluginBase):
+    async def hook(self, data: Event):
+        # Type narrowing for PyRight
+        if (
+            not isinstance(data.shortname, str)
+            or not isinstance(data.resource_type, ResourceType)
+            or not isinstance(data.attributes, dict)
+        ):
+            logger.error("invalid data at resource_folders_creation")
+            return
+
+        folders = []
+        if data.resource_type == ResourceType.user:
+            folders = [
+                ("personal", "people", data.shortname),
+                ("personal", f"people/{data.shortname}", "notifications"),
+                ("personal", f"people/{data.shortname}", "private"),
+                ("personal", f"people/{data.shortname}", "protected"),
+                ("personal", f"people/{data.shortname}", "public"),
+                ("personal", f"people/{data.shortname}", "inbox"),
+            ]
+        elif data.resource_type == ResourceType.space:
+            # sys_schemas = ["meta_schema", "folder_rendering"]
+            # for schema_name in sys_schemas:
+            #     await clone(
+            #         src_space=settings.management_space,
+            #         src_subpath="schema",
+            #         src_shortname=schema_name,
+            #         dest_space=data.shortname,
+            #         dest_subpath="schema",
+            #         dest_shortname=schema_name,
+            #         class_type=Schema,
+            #     )
+            if settings.active_data_db == "file":
+                async with RedisServices() as redis_services:
+                    await redis_services.create_indices(
+                        for_space=data.shortname,
+                        # for_schemas=sys_schemas,
+                        for_custom_indices=False,
+                        del_docs=False,
+                    )
+
+            # redis_update_plugin = RedisUpdatePlugin()
+            # for schema_name in sys_schemas:
+            #     await redis_update_plugin.hook(Event(
+            #         space_name=data.shortname,
+            #         subpath="schema",
+            #         shortname=schema_name,
+            #         action_type=ActionType.create,
+            #         resource_type=ResourceType.schema,
+            #         user_shortname=data.user_shortname
+            #     ))
+
+            folders = [(data.shortname, "/", "schema")]
+
+        for folder in folders:
+            existing_folder = await db.load_or_none(
+                space_name=folder[0],
+                subpath=folder[1],
+                shortname=folder[2],
+                class_type=Folder,
+                user_shortname=data.user_shortname,
+            )
+            if existing_folder is None:
+                await db.internal_save_model(
+                    space_name=folder[0],
+                    subpath=folder[1],
+                    meta=Folder(
+                        shortname=folder[2],
+                        is_active=True,
+                        owner_shortname=data.user_shortname,
+                    ),
+                )

dmart/plugins/system_notification_sender/config.json

@@ -0,0 +1,13 @@
+{
+  "shortname": "system_notification_sender",
+	"is_active": false,
+	"filters": {
+		"subpaths": ["__ALL__"],
+		"resource_types": ["__ALL__"],
+		"schema_shortnames": ["__ALL__"],
+		"actions": ["progress_ticket", "view", "create", "update", "delete", "attach", "move"]
+	},
+	"type": "hook",
+	"ordinal": 1,
+	"listen_time": "after"
+}

dmart/plugins/system_notification_sender/plugin.py

@@ -0,0 +1,188 @@
+from sys import modules as sys_modules
+from uuid import uuid4
+from models import api
+from models.enums import ContentType, QueryType
+from models.core import (
+    ActionType,
+    Content,
+    NotificationData,
+    Payload,
+    PluginBase,
+    Event,
+    Translation,
+)
+from utils.notification import NotificationManager
+from utils.helpers import camel_case, replace_message_vars
+from utils.settings import settings
+from fastapi.logger import logger
+from data_adapters.adapter import data_adapter as db
+
+
+class Plugin(PluginBase):
+    async def hook(self, data: Event):
+        """
+        after any action
+        1- get the matching SystemNotificationRequest for this action
+        2- generate list of users to send the notification to (based on the action entry):
+            2.1- entry.owner_shortname, entry.group.members?, and entry.collaborators?
+        3- send the notification to the list of generated users
+        """
+        # Type narrowing for PyRight
+        if not isinstance(data.shortname, str):
+            logger.warning(
+                "data.shortname is None and str is required at system_notification_sender"
+            )
+            return
+
+        if data.action_type == ActionType.delete and data.attributes.get("entry"):
+            entry = data.attributes["entry"].model_dump()
+        else:
+            entry = (
+                await db.load(
+                    data.space_name,
+                    data.subpath,
+                    data.shortname,
+                    getattr(sys_modules["models.core"], camel_case(data.resource_type)),
+                    data.user_shortname,
+                )
+            ).model_dump()
+            if entry["payload"] is not None:
+                try:
+                    entry["payload"]["body"] = await db.load_resource_payload(
+                        space_name=data.space_name,
+                        subpath=data.subpath,
+                        filename=entry["payload"]["body"],
+                        class_type=getattr(
+                            sys_modules["models.core"], camel_case(data.resource_type)
+                        ),
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to load payload for entry {data.space_name}/{data.subpath}/{data.shortname}: {e}"
+                    )
+        entry["space_name"] = data.space_name
+        entry["resource_type"] = str(data.resource_type)
+        entry["subpath"] = data.subpath
+        # 1- get the matching SystemNotificationRequests
+        total, matching_notification_requests = await db.query(api.Query(
+            type=QueryType.search,
+            retrieve_json_payload=True,
+            space_name="management",
+            subpath="notifications/system",
+            search=f"@payload.body.on_space:{data.space_name} @payload.body.on_subpath:{data.subpath.lstrip('/')} @payload.body.on_action:{data.action_type}",
+            limit=30,
+            offset=0
+        ), "dmart")
+        if total == 0:
+            return
+
+        sub_matching_notification_requests = matching_notification_requests[0].model_dump()
+        notification_dict = sub_matching_notification_requests
+        if (
+            "state" in entry
+            and notification_dict.get("on_state", "") != ""
+            and notification_dict["on_state"] != entry["state"]
+        ):
+            return
+
+
+        # 2- get list of subscribed users
+        notification_subscribers =[]
+        # if entry.get("collaborators", None):
+        #     notification_subscribers.extend(entry["collaborators"].values())
+
+        data_owner_shortname = entry["owner_shortname"]
+        if entry.get("owner_group_shortname", None):
+            group_users = await db.get_group_users(entry["owner_group_shortname"])
+            notification_subscribers.extend(group_users)        
+        if data_owner_shortname in notification_subscribers:
+            notification_subscribers.remove(data_owner_shortname)
+        users_objects: dict[str, dict] = {}
+
+        for subscriber in notification_subscribers:
+            users_objects[subscriber] = (await db.load(
+                settings.management_space,
+                settings.users_subpath,
+                subscriber,
+                getattr(sys_modules["models.core"], camel_case("user")),
+                data.user_shortname,
+            )).model_dump()
+        # 3- send the notification
+        notification_manager = NotificationManager()
+
+        formatted_req = await self.prepare_request(notification_dict, entry)
+        for receiver in set(notification_subscribers):
+            if not formatted_req["push_only"]:
+                notification_content = Content(
+                    shortname=str(uuid4())[:8],
+                    is_active=True,
+                    displayname=notification_dict["attributes"]["displayname"],
+                    description=notification_dict["attributes"]["description"],
+                    owner_shortname=receiver,
+                    payload=Payload(
+                        content_type=ContentType.json,
+                        body={
+                            "type": "system",
+                            "is_read": False,
+                            "priority": notification_dict["attributes"]["payload"]["body"]["priority"],
+                            "entry_space": entry["space_name"],
+                            "entry_subpath": entry["subpath"],
+                            "entry_shortname": entry["shortname"],
+                            "resource_type": entry["resource_type"],
+                            "created_by": data.user_shortname,
+                            "action_type": str(data.action_type)
+                        }
+                    )
+                )
+                await db.internal_save_model(
+                    space_name="personal",
+                    subpath=f"people/{receiver}/notifications",
+                    meta=notification_content
+                )
+
+            for platform in formatted_req["platforms"]:
+                await notification_manager.send(
+                    platform=platform,
+                    data=NotificationData(
+                        receiver=users_objects[receiver],
+                        title=formatted_req["title"],
+                        body=formatted_req["body"],
+                        image_urls=formatted_req["images_urls"],
+                        deep_link=notification_dict.get("deep_link", {}),
+                        entry_id=entry["shortname"],
+                    ),
+                )
+
+    async def prepare_request(self, notification_dict: dict, entry: dict) -> dict:
+        for locale in ["ar", "en", "ku"]:
+            if "displayname" in notification_dict:
+                notification_dict["displayname"][locale] = replace_message_vars(
+                    notification_dict["displayname"][locale], entry, locale
+                )
+            if "description" in notification_dict:
+                notification_dict["description"][locale] = replace_message_vars(
+                    notification_dict["description"][locale], entry, locale
+                )
+            
+        # Get Notification Request Images
+        attachments_path = (
+            settings.spaces_folder
+            / f"{settings.management_space}"
+            f"/{notification_dict['subpath']}/.dm/{notification_dict['shortname']}"
+        )
+        notification_attachments = await db.get_entry_attachments(
+            subpath=f"{notification_dict['subpath']}/{notification_dict['shortname']}",
+            attachments_path=attachments_path,
+        )
+        notification_images = {
+            "en": notification_attachments.get("media", {}).get("en"),
+            "ar": notification_attachments.get("media", {}).get("ar"),
+            "ku": notification_attachments.get("media", {}).get("ku"),
+        }
+        return {
+            "platforms": notification_dict.get("types", []),
+            "title": Translation(**notification_dict.get("displayname", {})),
+            "body": Translation(**notification_dict.get("description", {})),
+            "images_urls": Translation(**notification_images),
+            "push_only": notification_dict.get("push_only", False),
+        }

dmart/plugins/update_access_controls/config.json

@@ -0,0 +1,12 @@
+{
+  "shortname": "update_access_controls",
+	"is_active": true,
+	"filters": {
+		"subpaths": ["roles", "groups", "permissions", "users"],
+		"resource_types": ["group", "role", "permission", "user"],
+		"schema_shortnames": ["__ALL__"],
+		"actions": ["create", "update", "delete"]
+	},
+	"type": "hook",
+	"listen_time": "after"
+}

dmart/plugins/update_access_controls/plugin.py

@@ -0,0 +1,9 @@
+from models.core import PluginBase, Event
+from utils.access_control import access_control
+from utils.settings import settings
+
+
+class Plugin(PluginBase):
+    async def hook(self, data: Event):
+        if settings.active_data_db == "file":
+            await access_control.load_permissions_and_roles()

dmart/publish.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")"
+
+echo "Cleaning up dist/, build/, and temporary files..."
+rm -rf dist/ build/ *.egg-info dmart info.json
+
+echo "Building CXB..."
+CXB_DIR="../cxb"
+if [ -d "$CXB_DIR" ]; then
+    if [ -f "config.json" ]; then
+        cp "config.json" "$CXB_DIR/public/config.json"
+    elif [ -f "../config.json" ]; then
+        cp "../config.json" "$CXB_DIR/public/config.json"
+    elif [ ! -f "$CXB_DIR/public/config.json" ] && [ -f "$CXB_DIR/public/config.sample.json" ]; then
+        cp "$CXB_DIR/public/config.sample.json" "$CXB_DIR/public/config.json"
+    fi
+    pushd "$CXB_DIR"
+    yarn install
+    yarn build
+    popd
+
+    rm -rf cxb
+    cp -r "$CXB_DIR/dist/client" cxb
+    touch cxb/__init__.py
+else
+    echo "CXB directory not found at $CXB_DIR"
+    exit 1
+fi
+
+echo "Installing dependencies..."
+python3 -m pip install --upgrade pip build twine
+python3 -m pip install .
+if [ -d "requirements" ]; then
+    for req in requirements/*.txt; do
+        if [ -f "$req" ] && [[ "$req" != *"test.txt" ]]; then
+            python3 -m pip install -r "$req"
+        fi
+    done
+fi
+
+echo "Creating standalone bundle..."
+python3 bundler.py
+
+echo "Building dmart wheel and sdist..."
+python3 -m build
+
+rm -rf cxb
+ln -s "../cxb/dist/client" cxb
+
+if [ -f "dmart" ]; then
+    mv dmart dist/
+fi
+
+echo "Uploading to PyPI..."
+twine upload --non-interactive dist/*.whl

dmart/pylint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+declare -i RESULT=0
+export PYRIGHT_PYTHON_FORCE_VERSION=latest
+echo "Pyright ..."
+python -m pyright .
+RESULT+=$?
+echo "Ruff ..."
+python -m ruff check --exclude pytests --exclude alembic .
+RESULT+=$?
+echo "Mypy ..."
+python -m mypy --explicit-package-bases --warn-return-any --check-untyped-defs --exclude loadtest --exclude pytests --exclude alembic .
+RESULT+=$?
+
+echo "Result : $RESULT"
+exit $RESULT

dmart/pyrightconfig.json

@@ -0,0 +1,7 @@
+{
+  "exclude": [
+    "alembic",
+    "loadtest",
+    "pytests"
+  ]
+}

dmart/redis_connections.sh

@@ -0,0 +1,13 @@
+REDIS_HOST="$(./get_settings.py | jq -r .redis_host)"
+RESULT+=$?
+REDIS_PORT="$(./get_settings.py | jq -r .redis_port)"
+RESULT+=$?
+REDIS_PASSWORD="$(./get_settings.py | jq -r .redis_password)"
+[ -z $REDIS_PASSWORD ] || REDIS_PASSWORD="--no-auth-warning -a $REDIS_PASSWORD" 
+
+while true
+    do
+        redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${REDIS_PASSWORD}  info | grep connected_clients
+        redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${REDIS_PASSWORD}  info | grep blocked_clients
+        sleep 1
+    done

dmart/reload.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+declare -i RESULT=0
+source ./login_creds.sh
+RESULT+=$?
+CHECK_MODE="$(./get_settings.py | jq -r .active_data_db)"
+
+RESULT+=$?
+PORT="$(./get_settings.py | jq -r .listening_port)"
+RESULT+=$?
+# APP_URL="$(./get_settings.py | jq -r .app_url)"
+APP_URL="http://localhost:$PORT"
+
+if [[ "$CHECK_MODE" == "file" ]]; then
+REDIS_HOST="$(./get_settings.py | jq -r .redis_host)"
+RESULT+=$?
+REDIS_PORT="$(./get_settings.py | jq -r .redis_port)"
+RESULT+=$?
+REDIS_PASSWORD="$(./get_settings.py | jq -r .redis_password)"
+[ -z $REDIS_PASSWORD ] || REDIS_PASSWORD="--no-auth-warning -a $REDIS_PASSWORD" 
+time ./create_index.py --flushall
+RESULT+=$?
+fi
+
+(which systemctl > /dev/null && systemctl --user list-unit-files dmart.service > /dev/null  && systemctl --user restart dmart.service) || \
+( [[ -x "/etc/init.d/dmart" ]] && /etc/init.d/dmart restart )
+RESULT+=$?
+sleep 2
+# RESP=$(curl --write-out '%{http_code}' --silent --output /dev/null  "${APP_URL}")
+RESP="000"
+COUNTER=0
+while [ $RESP -ne "200" ]; do
+  sleep 1
+  COUNTER=$((COUNTER+1))
+  echo "Waiting for the server to come up ${RESP} ${COUNTER} seconds"
+  RESP=$(curl --write-out '%{http_code}' --silent --connect-timeout 0.5 --output /dev/null  "${APP_URL}")
+  [[ $COUNTER -ge 30 ]] && break
+done
+sleep 2
+
+TOKEN=$(curl -s "${APP_URL}/user/login" -H 'Content-Type: application/json' -d "${SUPERMAN}" | jq -r '.records[0].attributes.access_token')
+curl -s -H "Authorization: Bearer ${TOKEN}" -H "$CT" $APP_URL/user/profile | jq -r '.status'
+RESULT+=$?
+
+sleep 1
+curl -s -H "Authorization: Bearer ${TOKEN}" "${APP_URL}/user/profile" | jq '.records[0].attributes.roles'
+RESULT+=$?
+if [[ "$CHECK_MODE" == "file" ]]; then
+sleep 1
+redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${REDIS_PASSWORD} JSON.GET users_permissions_dmart | jq -R '.|fromjson|keys|length'
+# RESULT+=$?
+fi
+
+
+echo "Sum of exist codes = $RESULT" 
+exit $RESULT

dmart/run.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+python -m hypercorn main:app --config file:utils/hypercorn_config.py