dmart 0.1.9__py3-none-any.whl

pytests/json_to_db_migration_test.py

@@ -0,0 +1,237 @@
+import time
+from uuid import uuid4
+import pytest
+import os
+import json
+from pathlib import Path
+from sqlmodel import Session, create_engine, text, SQLModel
+from data_adapters.sql.create_tables import Attachments, Entries, Spaces, Histories
+from sqlalchemy.exc import OperationalError
+from utils.settings import settings
+
+
+def subpath_checker(subpath: str):
+    if subpath.endswith("/"):
+        subpath = subpath[:-1]
+    if not subpath.startswith("/"):
+        subpath = '/' + subpath
+    return subpath
+
+
+def connect_with_retry(engine, retries=5, delay=2):
+    """
+    Try to connect to the database with retries.
+    """
+    for attempt in range(retries):
+        try:
+            with engine.connect() as _:
+                print(f"Connected to the database on attempt {attempt + 1}")
+                return
+        except OperationalError as e:
+            print(f"Connection attempt {attempt + 1} failed: {e}")
+            time.sleep(delay)
+    raise Exception("Could not connect to the database after multiple attempts")
+
+
+@pytest.fixture(scope="module")
+def setup_database():
+    if settings.active_data_db == "file":
+        pytest.skip("Skipping test for file-based database")
+        return
+
+    # Use the settings to connect with the main `postgres` user
+    postgresql_url = f"{settings.database_driver.replace('+asyncpg','+psycopg')}://{settings.database_username}:{settings.database_password}@{settings.database_host}:{settings.database_port}"
+    engine = create_engine(f"{postgresql_url}/postgres", echo=False, isolation_level="AUTOCOMMIT")
+
+    # Create the database
+    with Session(engine) as session:
+        try:
+            session.exec(text(f"DROP DATABASE IF EXISTS {settings.database_name}"))
+            session.commit()
+            session.exec(text(f"CREATE DATABASE {settings.database_name}"))
+            session.commit()  # Ensure the transaction is fully committed
+            print(f"Database {settings.database_name} created successfully")
+        except Exception as e:
+            print(f"Database creation failed: {e}")
+
+    # Add a small delay to ensure the database is fully ready
+    time.sleep(2)
+
+    yield
+
+    # Drop the database after tests
+    with Session(engine) as session:
+        try:
+            session.exec(text(f"DROP DATABASE IF EXISTS {settings.database_name}"))
+            session.commit()
+            print(f"Database {settings.database_name} dropped successfully")
+        except Exception as e:
+            print(f"Database deletion failed: {e}")
+
+    engine.dispose()
+
+
+@pytest.fixture(scope="module")
+def setup_environment(setup_database):
+    if settings.active_data_db == "file":
+        pytest.skip("Skipping test for file-based database")
+        return
+
+    # Set the database name from settings
+    driver = settings.database_driver.replace('+asyncpg', '+psycopg')
+    postgresql_url = f"{driver}://{settings.database_username}:{settings.database_password}@{settings.database_host}:{settings.database_port}"
+    engine = create_engine(f"{postgresql_url}/{settings.database_name}", echo=False)
+
+    # Retry connecting to the newly created database
+    connect_with_retry(engine)
+
+    # Generate tables after ensuring connection
+    postgresql_url = f"{driver}://{settings.database_username}:{settings.database_password}@{settings.database_host}:{settings.database_port}/{settings.database_name}"
+    engine = create_engine(postgresql_url, echo=False)
+    SQLModel.metadata.create_all(engine)
+
+    yield engine
+
+    engine.dispose()
+
+
+def test_json_to_db_migration(setup_environment):
+    if settings.active_data_db == "file":
+        pytest.skip("Skipping test for file-based database")
+        return
+
+    engine = setup_environment
+
+    # Create a complex mock directory structure and files for different entry types
+    os.makedirs('/tmp/test_space/.dm', exist_ok=True)
+    with open('/tmp/test_space/.dm/meta.space.json', 'w') as f:
+        json.dump({"key": "value"}, f)
+
+    # Create more directories and files for the migration
+    os.makedirs('/tmp/test_space/dir1', exist_ok=True)
+    with open('/tmp/test_space/dir1/history.jsonl', 'w') as f:
+        f.write(json.dumps({"key": "history"}) + '\n')
+
+    # Create attachments folder and files
+    os.makedirs('/tmp/test_space/dir1/attachments', exist_ok=True)
+    with open('/tmp/test_space/dir1/attachments/meta.attachments.json', 'w') as f:
+        json.dump({
+            "uuid": str(uuid4()),
+            "space_name": "test_space",
+            "subpath": "/dir1",
+            "acl": [],
+            "relationships": [],
+            "payload": {"body": "attachment content"}
+        }, f)
+
+    # Create ticket-related file
+    with open('/tmp/test_space/dir1/meta.ticket.json', 'w') as f:
+        json.dump({
+            "state": "open",
+            "is_open": True,
+            "reporter": "user1",
+            "subpath": "/dir1/ticket"
+        }, f)
+
+    # Create user meta file
+    with open('/tmp/test_space/.dm/meta.user.json', 'w') as f:
+        json.dump({
+            "resource_type": "user",
+            "firebase_token": "firebase_token",
+            "language": "en"
+        }, f)
+
+    # Create role meta file
+    with open('/tmp/test_space/.dm/meta.role.json', 'w') as f:
+        json.dump({
+            "resource_type": "role",
+            "permissions": ["read", "write"]
+        }, f)
+
+    # Create permission meta file
+    with open('/tmp/test_space/.dm/meta.permission.json', 'w') as f:
+        json.dump({
+            "resource_type": "permission",
+            "subpaths": {"read": "/read", "write": "/write"},
+            "resource_types": ["user", "role"]
+        }, f)
+
+    # Run the migration script
+    try:
+        with Session(engine) as session:
+            for root, dirs, _ in os.walk('/tmp/test_space'):
+                tmp = root.replace('/tmp/test_space', '')
+                if tmp == '':
+                    continue
+                if tmp[0] == '/':
+                    tmp = tmp[1:]
+                space_name = tmp.split('/')[0]
+                subpath = '/'.join(tmp.split('/')[1:])
+                if space_name == '..':
+                    continue
+
+                if space_name.startswith('.git'):
+                    continue
+
+                if subpath == '' or subpath == '/':
+                    subpath = '/'
+                    p = os.path.join(root, '.dm', 'meta.space.json')
+                    entry = {}
+                    if Path(p).is_file():
+                        entry = json.load(open(p))
+                        entry['space_name'] = space_name
+                        entry['subpath'] = '/'
+                        session.add(Spaces.model_validate(entry))
+                    continue
+
+                subpath = subpath.replace('.dm', '')
+                if subpath != '/' and subpath.endswith('/'):
+                    subpath = subpath[:-1]
+
+                if subpath == '':
+                    subpath = '/'
+
+                for dir in dirs:
+                    for file in os.listdir(os.path.join(root, dir)):
+                        if not file.startswith('meta'):
+                            if file == 'history.jsonl':
+                                lines = open(os.path.join(root, dir, file), 'r').readlines()
+                                for line in lines:
+                                    history = json.loads(line)
+                                    history['shortname'] = dir
+                                    history['space_name'] = space_name
+                                    history['subpath'] = subpath_checker(subpath)
+                                    session.add(Histories.model_validate(history))
+                            continue
+
+                        p = os.path.join(root, dir, file)
+                        if Path(p).is_file():
+                            if 'attachments' in p:
+                                _attachment = json.load(open(os.path.join(root, dir, file)))
+                                _attachment['space_name'] = space_name
+                                _attachment['uuid'] = _attachment.get('uuid', uuid4())
+                                _attachment['subpath'] = subpath_checker(_attachment['subpath'])
+                                session.add(Attachments.model_validate(_attachment))
+                            elif file.endswith('.json'):
+                                entry = json.load(open(p))
+                                entry['space_name'] = space_name
+                                entry['subpath'] = subpath_checker(subpath)
+                                session.add(Entries.model_validate(entry))
+            session.commit()
+        assert True  # Assert that the migration completes without error
+    except Exception as e:
+        print(f"Migration failed: {e}")
+        assert False  # Fail the test if there is any exception
+
+    # Clean up the mock directory structure
+    os.remove('/tmp/test_space/.dm/meta.space.json')
+    os.remove('/tmp/test_space/dir1/history.jsonl')
+    os.remove('/tmp/test_space/dir1/attachments/meta.attachments.json')
+    os.remove('/tmp/test_space/dir1/meta.ticket.json')
+    os.remove('/tmp/test_space/.dm/meta.user.json')
+    os.remove('/tmp/test_space/.dm/meta.role.json')
+    os.remove('/tmp/test_space/.dm/meta.permission.json')
+    os.rmdir('/tmp/test_space/dir1/attachments')
+    os.rmdir('/tmp/test_space/.dm')
+    os.rmdir('/tmp/test_space/dir1')
+    os.rmdir('/tmp/test_space')

pytests/service_test.py

@@ -0,0 +1,26 @@
+from api.user.service import gen_alphanumeric
+import pytest
+
+@pytest.mark.run(order=9)
+def test_gen_alphanumeric_length():
+    # Test default length
+    result = gen_alphanumeric()
+    assert len(result) == 16, "Default length should be 16"
+
+    # Test custom length
+    length = 32
+    result = gen_alphanumeric(length)
+    assert len(result) == length, f"Length should be {length}"
+
+
+@pytest.mark.run(order=9)
+def test_gen_alphanumeric_characters():
+    result = gen_alphanumeric()
+    assert all(c.isalnum() for c in result), "Result should only contain alphanumeric characters"
+
+
+@pytest.mark.run(order=9)
+def test_gen_alphanumeric_unique():
+    num_samples = 100
+    samples = {gen_alphanumeric() for _ in range(num_samples)}
+    assert len(samples) == num_samples, "Generated strings should be unique"

pytests/test_info.py

@@ -0,0 +1,55 @@
+from httpx import AsyncClient
+from pytests.base_test import get_superman_cookie
+from fastapi import status
+import pytest
+from utils.internal_error_code import InternalErrorCode
+from utils.jwt import sign_jwt
+from utils.settings import settings
+
+
+
+@pytest.mark.run(order=6)
+@pytest.mark.anyio
+async def test_info_me(client: AsyncClient) -> None:
+
+    client.cookies.set("auth_token", await get_superman_cookie(client))
+    response = await client.get("/info/me")
+    assert response.status_code == status.HTTP_200_OK
+    json_response = response.json()
+    assert json_response["status"] == "success"
+
+@pytest.mark.run(order=6)
+@pytest.mark.anyio
+async def test_info_manifest(client: AsyncClient) -> None:
+
+        client.cookies.set("auth_token", await get_superman_cookie(client))
+        response = await client.get("/info/manifest")
+        assert response.status_code == status.HTTP_200_OK
+        json_response = response.json()
+        assert json_response["status"] == "success"
+
+
+@pytest.mark.run(order=6)
+@pytest.mark.anyio
+async def test_get_settings_should_pass(client: AsyncClient) -> None:
+    client.cookies.set("auth_token", await get_superman_cookie(client))
+    response = await client.get("/info/settings")
+    assert response.status_code == status.HTTP_200_OK
+
+
+# @pytest.mark.run(order=6)
+# @pytest.mark.anyio
+# async def test_in_loop_tasks(client: AsyncClient) -> None:
+#     client.cookies.set("auth_token", await get_superman_cookie(client))
+#     response = await client.get("/info/in-loop-tasks")
+#     assert response.status_code == status.HTTP_200_OK
+#     json_response = response.json()
+#     assert json_response["status"] == "success"
+#     assert "tasks_count" in json_response["attributes"]
+#     assert isinstance(json_response["attributes"]["tasks_count"], int)
+#     assert "tasks" in json_response["attributes"]
+#     assert isinstance(json_response["attributes"]["tasks"], list)
+#     for task in json_response["attributes"]["tasks"]:
+#         assert "name" in task
+#         assert "coroutine" in task
+#         assert "stack" in task

pytests/test_status.py

@@ -0,0 +1,15 @@
+
+# from pytests.base_test import  client
+from fastapi import status
+from httpx import AsyncClient
+import pytest
+
+# @pytest.mark.asyncio(scope="session")
+@pytest.mark.anyio
+async def test_sanity(client: AsyncClient) -> None:
+#    async with my_client as client:
+        response = await client.get("/")
+        assert response.status_code == status.HTTP_200_OK
+        json_response = response.json()
+        assert json_response["status"] == "success"
+

run_notification_campaign.py

@@ -0,0 +1,98 @@
+import argparse
+import asyncio
+from importlib.util import find_spec, module_from_spec
+import sys
+
+from models.core import Content, Event
+from models.enums import ActionType
+from data_adapters.adapter import data_adapter as db
+from utils.settings import settings
+
+CUSTOM_PLUGINS_PATH = settings.spaces_folder / "custom_plugins"
+
+# Allow python to search for modules inside the custom plugins
+# be including the path to the parent folder of the custom plugins to sys.path
+back_out_of_project = 2
+back_to_spaces = 0
+
+for part in CUSTOM_PLUGINS_PATH.parts:
+    if part == "..":
+        back_to_spaces += 1
+
+if __file__.endswith(".pyc"):
+    back_out_of_project += 1
+
+sys.path.append(
+    "/".join(__file__.split("/")[:-(back_out_of_project+back_to_spaces)]) + 
+    "/" +
+    "/".join(CUSTOM_PLUGINS_PATH.parts[back_to_spaces:-1])
+)
+
+def load_notification_plugin():
+    # Load the plugin module
+    plugin_path = CUSTOM_PLUGINS_PATH / 'send_notification'
+    
+    config_file_path = plugin_path / 'config.json'
+    plugin_file_path = plugin_path / 'plugin.py'
+    if(
+        not config_file_path.is_file() or
+        not plugin_file_path.is_file()
+    ):
+        return None
+
+
+    module_name = f"{CUSTOM_PLUGINS_PATH.parts[-1]}.send_notification.plugin"
+    spec = find_spec(module_name)
+    if not spec:
+        return None
+    module = module_from_spec(spec)
+    sys.modules[module_name] = module
+    return getattr(module, "Plugin")()
+    
+    
+async def main(space, subpath, shortname):
+    notification_payload = db.load_resource_payload(
+        space_name=space,
+        subpath=subpath,
+        class_type=Content,
+        filename=f"{shortname}.json",
+    )
+    
+    if not notification_payload:
+        print("The notification entry is not found")
+        return
+    
+    event_data = Event(
+        space_name=space,
+        subpath=subpath,
+        shortname=shortname,
+        action_type=ActionType.create,
+        attributes={
+            "payload": {
+                "body": notification_payload
+            }
+        },
+        user_shortname="__SYSTEM__"
+    )
+    
+    
+    plugin_obj = load_notification_plugin()
+    if not plugin_obj:
+        print("The plugin is not found")
+        return
+    
+    await plugin_obj.hook(event_data)
+    
+    
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Execute the custom_plugins/send_notification plugin",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--space", help="The space where the notification request and user records are stored")
+    parser.add_argument("--notification-subpath", help="The subpath of the notification request")
+    parser.add_argument("--notification-shortname", help="The shortname of the notification request")
+
+    args = parser.parse_args()
+
+    asyncio.run(main(args.space, args.notification_subpath, args.notification_shortname))

scheduled_notification_handler.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env -S BACKEND_ENV=config.env python3
+from datetime import datetime, timedelta
+
+from models.api import Query
+from models.core import Content, Notification, NotificationData, Translation
+from data_adapters.adapter import data_adapter as db
+from utils.notification import NotificationManager
+from utils.settings import settings
+from fastapi.logger import logger
+import asyncio
+
+
+async def trigger_admin_notifications() -> None:
+    from_time = int((datetime.now() - timedelta(minutes=15)).timestamp() * 1000)
+    to_time = int(datetime.now().timestamp() * 1000)
+    total, admin_notifications = await db.query(Query(
+        space_name=settings.management_space, schema_name="admin_notification_request",
+        search=f"@subpath:/notifications/admin (-@status:finished) @scheduled_at:[{from_time} {to_time}]",
+        filters={}, limit=10000, offset=0)
+    )
+
+    if total == 0:
+        return
+
+    notification_manager = NotificationManager()
+    for notification in admin_notifications:
+        notification_dict = notification.model_dump()
+        formatted_req = await prepare_request(notification_dict)
+
+        # Get notification receivers users
+        search_criteria = notification_dict.get('msisdns_search_string')
+        if not search_criteria:
+            search_criteria = '@msisdn:' + '|'.join(notification_dict.get('msisdns', ""))
+
+        total, receivers = await db.query(Query(
+            space_name=settings.management_space, schema_name="user",
+            search=f"@subpath:users {search_criteria}",
+            filters={}, limit=10000, offset=0)
+        )
+
+        if total == 0:
+            continue
+
+        # Try to send the notification
+        # and update the notification status to finished
+        formatted_req = await prepare_request(notification_dict)
+        try:
+            for receiver_data in receivers:
+                if not formatted_req["push_only"]:
+                    notification_obj = await Notification.from_request(
+                        notification_dict
+                    )
+                    await db.internal_save_model(
+                        space_name="personal",
+                        subpath=f"people/{receiver_data.shortname}/notifications",
+                        meta=notification_obj,
+                    )
+
+                for platform in formatted_req["platforms"]:
+                    await notification_manager.send(
+                        platform=platform,
+                        data=NotificationData(
+                            receiver=receiver_data.to_dict(),
+                            title=formatted_req["title"],
+                            body=formatted_req["body"],
+                            image_urls=formatted_req["images_urls"],
+                        ),
+                    )
+
+            notification_meta = await db.load_or_none(
+                settings.management_space,
+                notification_dict["subpath"],
+                notification_dict["shortname"],
+                Content,
+                notification_dict["owner_shortname"],
+            )
+
+            if notification_meta is None:
+                return
+
+            await db.internal_sys_update_model(
+                settings.management_space,
+                notification_dict["subpath"],
+                notification_meta,
+                {"status": "finished"},
+            )
+        except Exception as e:
+            logger.error(
+                f"Error at sending/updating admin based notification: {e.args}"
+            )
+            pass
+
+
+async def prepare_request(notification_dict) -> dict:
+    # Get Notification Request Images
+    attachments_path = (
+        settings.spaces_folder
+        / f"{settings.management_space}"
+        f"/{notification_dict['subpath']}/.dm/{notification_dict['shortname']}"
+    )
+    notification_attachments = await db.get_entry_attachments(
+        subpath=f"{notification_dict['subpath']}/{notification_dict['shortname']}",
+        attachments_path=attachments_path,
+    )
+    notification_images = {
+        "en": notification_attachments.get("media", {}).get("en"),
+        "ar": notification_attachments.get("media", {}).get("ar"),
+        "ku": notification_attachments.get("media", {}).get("ku"),
+    }
+
+    return {
+        "platforms": notification_dict["types"],
+        "title": Translation(**notification_dict["displayname"]),
+        "body": Translation(**notification_dict["description"]),
+        "images_urls": Translation(**notification_images),
+        "push_only": notification_dict.get("push_only", False),
+    }
+
+
+if __name__ == "__main__":
+    asyncio.run(trigger_admin_notifications())