dmart 1.4.40.post8__py3-none-any.whl

dmart/utils/logger.py

@@ -0,0 +1,167 @@
+import re
+import json
+import logging
+import os
+
+from utils.settings import settings
+import socket
+
+
+class JSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        return str(o)
+
+
+COMBINED_SENSITIVE_PATTERN = re.compile(
+    r'(?i)(?:'
+    r'(your\s+otp\s+code\s+is\s+)\d{4,8}|'
+    r'(otp\s+for\s+\d+\s+is\s+)\d{4,8}|'
+    r'(zip\s+pw\s+for\s+\d+\s+is\s+)[a-f0-9]{6,}|'
+    r'(your\s+password\s+for\s+the\s+export\s+zip\s+is\s+)[a-f0-9]{6,}|'
+    r'(invitation=)[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|'
+    r'("pin"\s*:\s*)"[^"]*"|'
+    r'("authorization"\s*:\s*)"[^"]*"|'
+    r'("auth_token"\s*:\s*)"[^"]*"|'
+    r'("firebase_token"\s*:\s*)"[^"]*"|'
+    r'("evd-device-id"\s*:\s*)"[^"]*"|'
+    r'("access_token"\s*:\s*)"[^"]*"|'
+    r'("cookie"\s*:\s*)"[^"]*"|'
+    r'("set-cookie"\s*:\s*)"[^"]*"'
+    r')'
+)
+
+
+SENSITIVE_KEYWORDS = ("authorization", "token", "password", "otp", "pin", "cookie", "auth", "firebase_token",
+                      "evd-device-id")
+
+
+def mask_replacement(match):
+    """Replace matched groups with a general mask."""
+    groups = match.groups()
+    for i, group in enumerate(groups):
+        if group is not None:
+            # For JSON format: "key": "value" → "key": "******"
+            if '"' in group:
+                return group + '"' + ('*' * 6) + '"'
+            # For plain format: key: value → key: ******
+            else:
+                return group + ('*' * 6)
+    return match.group(0)
+
+
+def mask_sensitive_data_string(log_string: str) -> str:
+    """Mask sensitive data only if keywords present."""
+    if not isinstance(log_string, str):
+        return log_string
+    # Quick keyword check first (very fast)
+    if not any(keyword in log_string.lower() for keyword in SENSITIVE_KEYWORDS):
+        return log_string
+    # Only apply regex if keywords found
+    return COMBINED_SENSITIVE_PATTERN.sub(mask_replacement, log_string)
+
+
+class CustomFormatter(logging.Formatter):
+    """
+    Emits one JSON line with this exact key order:
+    correlation_id, time, level, message, props, thread, process
+    """
+    def __init__(self) -> None:
+        log_dir = os.path.dirname(settings.log_file)
+        if not os.path.exists(log_dir):
+            os.mkdir(log_dir)
+        super().__init__()
+
+    def format(self, record):
+        correlation_id = getattr(record, "correlation_id", "")
+        if correlation_id == "ROOT" and getattr(record, "props", None):
+            correlation_id = getattr(record, "props", {})\
+                .get("response", {}).get("headers", {}).get("x-correlation-id", "")
+
+        props = getattr(record, "props", {})
+
+        # Extract hostname
+        hostname = socket.gethostname()
+
+        data = {
+            "hostname": hostname,
+            "correlation_id": correlation_id,
+            "time": self.formatTime(record),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "props": props,
+            "thread": record.threadName,
+            "process": record.process,
+        }
+        try:
+            log_string = json.dumps(data, cls=JSONEncoder)
+            masked_log = mask_sensitive_data_string(log_string)
+            return masked_log
+        except Exception as e:
+            return json.dumps({"error": str(e), "message": record.getMessage()})
+
+
+logging_schema : dict = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "filters": {
+        "correlation_id": {
+            "()": "asgi_correlation_id.CorrelationIdFilter",
+            "uuid_length": 32,
+            "default_value": "ROOT",
+        },
+    },
+    "formatters": {
+        "json": {"()": "utils.logger.CustomFormatter"},
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "filters": ["correlation_id"],
+            "level": "INFO",
+            "formatter": "json",
+            "stream": "ext://sys.stdout",  # Default is stderr
+        },
+        "file": {
+            "class": "concurrent_log_handler.ConcurrentRotatingFileHandler",
+            "filters": ["correlation_id"],
+            "filename": settings.log_file,
+            "backupCount": 5,
+            "maxBytes": 0x10000000,
+            "use_gzip": False,
+            "formatter": "json",
+        },
+    },
+    "root": {
+        "handlers": settings.log_handlers,
+        "level": "INFO",
+    },
+    "loggers": {
+        "fastapi": {
+            "handlers": settings.log_handlers,
+            "level": logging.INFO,
+            "propagate": False,
+        },
+        "hypercorn": {
+            "handlers": settings.log_handlers,
+            "level": logging.INFO,
+            "propagate": False,
+        },
+        "hypercorn.error": {
+            "handlers": settings.log_handlers,
+            "level": logging.INFO,
+            "propagate": False,
+        },
+        "hypercorn.access": {
+            "handlers": settings.log_handlers,
+            "level": logging.INFO,
+            "propagate": False,
+        },
+    },
+}
+
+
+def changeLogFile(log_file: str | None = None) -> None:
+    global logging_schema
+    if (log_file and "handlers" in logging_schema and "file" in logging_schema["handlers"] 
+        and "filename" in logging_schema["handlers"]["file"]):
+        logging_schema["handlers"]["file"]["filename"] = log_file

dmart/utils/middleware.py

@@ -0,0 +1,99 @@
+from contextvars import ContextVar
+from typing import Any
+from starlette.types import ASGIApp, Receive, Scope, Send
+from starlette.requests import Request
+from utils.internal_error_code import InternalErrorCode
+from utils.settings import settings
+import models.api as api
+from fastapi import status
+
+REQUEST_DATA_CTX_KEY = "request_data"
+
+_request_data_ctx_var: ContextVar[dict] = ContextVar(REQUEST_DATA_CTX_KEY, default={})
+
+def get_request_data() -> dict:
+    return _request_data_ctx_var.get()
+
+class CustomRequestMiddleware:
+    def __init__(
+        self,
+        app: ASGIApp,
+    ) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] not in ["http", "websocket"]:
+           try:
+               await self.app(scope, receive, send)
+           except Exception as _:
+               return
+
+
+        request = Request(scope, receive)
+        request_headers = {}
+        for k,v in request.headers.items():
+            if k in ['cookie', 'authorization']:
+                continue
+            request_headers[k] = v
+
+        request_data = _request_data_ctx_var.set({
+            "request_headers": request_headers,
+        })
+
+        await self.app(scope, receive, send)
+
+        _request_data_ctx_var.reset(request_data)
+
+
+class ChannelMiddleware:
+    def __init__(
+        self,
+        app: ASGIApp,
+    ) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] not in ["http", "websocket"] or not settings.enable_channel_auth:
+            await self.app(scope, receive, send)
+            return
+
+        request = Request(scope, receive)
+        channel_key = request.headers.get("x-channel-key")
+        if not channel_key:
+            for channel in settings.channels:
+                for pattern in channel["allowed_api_patterns"]:
+                    if pattern.search(request.scope['path']):
+                        raise api.Exception(
+                            status_code=status.HTTP_403_FORBIDDEN,
+                            error=api.Error(
+                                type="channel_auth", code=InternalErrorCode.NOT_ALLOWED, message="Requested method or path is forbidden"
+                            ),
+                        )
+            await self.app(scope, receive, send)
+            return
+
+        request_channel: dict[str, Any] | None = None
+        for channel in settings.channels:
+            if channel_key in channel.get("keys", []):
+                request_channel = channel
+                break
+
+        if not request_channel:
+            raise api.Exception(
+                status_code=status.HTTP_403_FORBIDDEN,
+                error=api.Error(
+                    type="channel_auth", code=InternalErrorCode.NOT_ALLOWED, message="Requested method or path is forbidden [2]"
+                ),
+            )
+
+        for pattern in request_channel["allowed_api_patterns"]:
+            if pattern.search(request.scope['path']):
+                await self.app(scope, receive, send)
+                return
+            
+        raise api.Exception(
+            status_code=status.HTTP_403_FORBIDDEN,
+            error=api.Error(
+                type="channel_auth", code=InternalErrorCode.NOT_ALLOWED, message="Requested method or path is forbidden [3]"
+            ),
+        )

dmart/utils/notification.py

@@ -0,0 +1,75 @@
+from abc import ABC, abstractmethod
+from importlib.util import find_spec, module_from_spec
+import json
+import sys
+from typing import Any
+from pathlib import Path
+
+from models.core import NotificationData
+from utils.settings import settings
+from models.core import User
+from data_adapters.adapter import data_adapter as db
+from fastapi.logger import logger
+
+
+class Notifier(ABC):
+    
+    @abstractmethod
+    async def send(self, data: NotificationData) -> bool:
+        pass
+
+    async def _load_user(self, shortname: str) -> Any:
+        if not hasattr(self, "user"):
+
+            self.user = await db.load(
+                space_name=settings.management_space,
+                subpath=settings.users_subpath,
+                shortname=shortname,
+                class_type=User,
+                user_shortname="__system__",
+            )
+        return self.user
+
+
+class NotificationManager:
+
+    notifiers: dict[str, Notifier] = {}
+
+    def __init__(self) -> None:
+        # Load the notifiers depending on config/notification.json file
+        if not self.notifiers:
+            config_path = Path(__file__).resolve().parent.parent / "config/notification.json"
+            if config_path.exists():
+                with open(config_path) as conf_file:
+                    conf_data = json.load(conf_file)
+
+                for platform, data in conf_data.items():
+                    if not data["active"]:
+                        continue
+
+                    module_name = data["module"]
+                    module_specs = find_spec(module_name)
+                    if not module_specs or not module_specs.loader:
+                        continue
+                    module = module_from_spec(module_specs)
+                    sys.modules[module_name] = module
+                    module_specs.loader.exec_module(module)
+                    self.notifiers[platform] = getattr(module, data["class"])()
+
+    async def send(self, platform, data: NotificationData) -> bool:
+        if platform not in self.notifiers:
+            return False
+        try:
+            await self.notifiers[platform].send(data)
+            return True
+        except Exception as e:
+            logger.warning(
+                "Notification",
+                extra={
+                    "props": {
+                        "title": f"FAIL at {self.notifiers[platform]}.send",
+                        "message": str(e),
+                    }
+                },
+            )
+            return False

dmart/utils/password_hashing.py

@@ -0,0 +1,16 @@
+from argon2 import PasswordHasher
+
+ph = PasswordHasher(
+    memory_cost=102400,
+    time_cost=1,
+    parallelism=8
+)
+
+def verify_password(plain_password: str, hashed_password: str):
+    try:
+        return ph.verify(hashed_password, plain_password)
+    except Exception:
+        return False
+
+def hash_password(password: str):
+    return ph.hash(password)

dmart/utils/plugin_manager.py

@@ -0,0 +1,202 @@
+import asyncio
+import os
+import sys
+from importlib.util import find_spec, module_from_spec
+from inspect import iscoroutine
+from pathlib import Path
+import aiofiles
+from fastapi import Depends, FastAPI
+from fastapi.logger import logger
+from models.core import (
+    ActionType,
+    PluginWrapper,
+    PluginBase,
+    Event,
+    EventFilter,
+    EventListenTime,
+)
+from models.enums import ResourceType, PluginType
+from utils.settings import settings
+
+CUSTOM_PLUGINS_PATH = settings.spaces_folder / "custom_plugins"
+
+# Allow python to search for modules inside the custom plugins
+# by including the path to the parent folder of the custom plugins to sys.path
+if CUSTOM_PLUGINS_PATH.parent.exists():
+    sys.path.append(str(CUSTOM_PLUGINS_PATH.parent.resolve()))
+
+class PluginManager:
+
+    plugins_wrappers: dict[
+        ActionType, list[PluginWrapper]
+    ] = {}  # {action_type: list_of_plugins_wrappers]}
+
+    async def load_plugins(self, app: FastAPI, capture_body):
+        # Load core plugins
+        current = Path(__file__).resolve().parent
+        parent = current.parent
+        path = parent / "plugins"
+        if path.is_dir():
+            await self.load_path_plugins(path, app, capture_body)
+
+        # Load custom plugins
+        path = CUSTOM_PLUGINS_PATH
+        if path.is_dir():
+            await self.load_path_plugins(path, app, capture_body)
+        self.sort_plugins()
+
+
+    async def load_path_plugins(self, path: Path, app: FastAPI, capture_body):
+
+        plugins_iterator = os.scandir(path)
+        for plugin_path in plugins_iterator:
+            config_file_path = Path(f"{plugin_path.path}/config.json")
+            plugin_file_path = Path(f"{plugin_path.path}/plugin.py")
+            if(
+                not config_file_path.is_file() or
+                not plugin_file_path.is_file()
+            ):
+                continue
+
+            # Load plugin config file
+            async with aiofiles.open(config_file_path, "r") as config_file:
+                plugin_wrapper: PluginWrapper = PluginWrapper.model_validate_json(
+                    await config_file.read()
+                )
+            plugin_wrapper.shortname = plugin_path.name
+            if not plugin_wrapper.is_active:
+                continue
+            # Load the plugin module
+            module_name = f"{path.parts[-1]}.{plugin_wrapper.shortname}.plugin"
+            spec = find_spec(module_name)
+            if not spec:
+                continue
+            module = module_from_spec(spec)
+            sys.modules[module_name] = module
+            if not spec.loader:
+                continue
+            spec.loader.exec_module(module)
+            try:
+                # Register the API plugin routes
+                if plugin_wrapper.type == PluginType.api:
+                    app.include_router(
+                        module.router,
+                        prefix=f"/{plugin_wrapper.shortname}",
+                        tags=[plugin_wrapper.shortname],
+                        dependencies=[Depends(capture_body)],
+                    )
+
+                # Add the Hook Plugin to the loaded plugins
+                elif plugin_wrapper.type == PluginType.hook:
+                    plugin_wrapper.object = getattr(module, "Plugin")()
+
+                    self.store_plugin_in_its_action_dict(plugin_wrapper)
+            except Exception as e:
+                logger.error(
+                    f"PLUGIN_ERROR, PLUGIN API {plugin_wrapper.shortname} Failed to load, error: {e.args}"
+                )
+
+        plugins_iterator.close()
+
+    def store_plugin_in_its_action_dict(self, plugin_wrapper: PluginWrapper):
+        if plugin_wrapper.filters:
+            for action in plugin_wrapper.filters.actions:
+                self.plugins_wrappers.setdefault(action, []).append(plugin_wrapper)
+
+    def sort_plugins(self):
+        """Sort plugins based on plugin_wrapper.ordinal"""
+
+        for action_type, plugins in self.plugins_wrappers.items():
+            self.plugins_wrappers[action_type] = sorted(
+                plugins, key=lambda x: x.ordinal
+            )
+
+    def matched_filters(self, plugin_filters: EventFilter, event: Event):
+        formats_of_subpath = [event.subpath]
+        if event.subpath and event.subpath[0] == "/":
+            formats_of_subpath.append(event.subpath[1:])
+        else:
+            formats_of_subpath.append(f"/{event.subpath}")
+
+        if "__ALL__" not in plugin_filters.subpaths and not any(
+            subpath in plugin_filters.subpaths for subpath in formats_of_subpath
+        ):
+            return False
+
+        if event.resource_type == ResourceType.content and (
+            "__ALL__" not in plugin_filters.schema_shortnames
+            and event.schema_shortname not in plugin_filters.schema_shortnames
+        ):
+            return False
+
+        if (
+            plugin_filters.resource_types
+            and "__ALL__" not in plugin_filters.resource_types
+            and event.resource_type not in plugin_filters.resource_types
+        ):
+            return False
+
+        return True
+
+    async def before_action(self, event: Event):
+        if event.action_type not in self.plugins_wrappers:
+            return
+
+        from data_adapters.adapter import data_adapter as db
+        space = await db.fetch_space(event.space_name)
+        if space is None:
+            return
+        space_plugins = space.active_plugins
+
+        loop = asyncio.get_event_loop()
+        for plugin_model in self.plugins_wrappers[event.action_type]:
+            if (
+                plugin_model.shortname in space_plugins
+                and plugin_model.listen_time == EventListenTime.before
+                and plugin_model.filters
+                and self.matched_filters(plugin_model.filters, event)
+            ):
+                try:
+                    object = plugin_model.object
+                    if isinstance(object, PluginBase):
+                        plugin_execution = object.hook(event)
+                        if iscoroutine(plugin_execution):
+                            loop.create_task(plugin_execution)
+                except Exception as e:
+                    # print(f"Plugin:{plugin_model}:{str(e)}")
+                    logger.error(f"Plugin:{plugin_model}:{str(e)}")
+
+    async def after_action(self, event: Event):
+        if event.action_type not in self.plugins_wrappers:
+            return
+
+        from data_adapters.adapter import data_adapter as db
+        space = await db.fetch_space(event.space_name)
+        if space is None:
+            return
+        space_plugins = space.active_plugins
+
+        loop = asyncio.get_event_loop()
+        _plugin_model = None
+        try:
+            for plugin_model in self.plugins_wrappers[event.action_type]:
+                _plugin_model = plugin_model
+                if (
+                    plugin_model.shortname in space_plugins
+                    and plugin_model.listen_time == EventListenTime.after
+                    and plugin_model.filters
+                    and self.matched_filters(plugin_model.filters, event)
+                ):
+                    try:
+                        object = plugin_model.object
+                        if isinstance(object, PluginBase):
+                            plugin_execution = object.hook(event)
+                            if iscoroutine(plugin_execution):
+                                loop.create_task(plugin_execution)
+                    except Exception as e:
+                        logger.error(f"Plugin:{plugin_model}:{str(e)}")
+        except Exception as e:
+            logger.error(f"Plugin:{_plugin_model}:{str(e)}")
+
+
+plugin_manager = PluginManager()

dmart/utils/query_policies_helper.py

@@ -0,0 +1,128 @@
+from models.enums import ResourceType, ConditionType
+from utils.settings import settings
+
+def generate_query_policies(
+        space_name: str,
+        subpath: str,
+        resource_type: str,
+        is_active: bool,
+        owner_shortname: str,
+        owner_group_shortname: str | None,
+        entry_shortname: str | None = None,
+) -> list:
+    subpath_parts = ["/"]
+    subpath_parts += subpath.strip("/").split("/")
+
+    if resource_type == ResourceType.folder and entry_shortname:
+        subpath_parts.append(entry_shortname)
+
+    query_policies: list = []
+    full_subpath = ""
+    for subpath_part in subpath_parts:
+        full_subpath += subpath_part
+        query_policies.append(
+            f"{space_name}:{full_subpath.strip('/')}:{resource_type}:{str(is_active).lower()}:{owner_shortname}"
+        )
+        if owner_group_shortname is None:
+            query_policies.append(
+                f"{space_name}:{full_subpath.strip('/')}:{resource_type}:{str(is_active).lower()}"
+            )
+        else:
+            query_policies.append(
+                f"{space_name}:{full_subpath.strip('/')}:{resource_type}:{str(is_active).lower()}:{owner_group_shortname}"
+            )
+
+        full_subpath_parts = full_subpath.split("/")
+        if len(full_subpath_parts) > 1:
+            subpath_with_magic_keyword = (
+                    "/".join(full_subpath_parts[:1]) + "/" + settings.all_subpaths_mw
+            )
+            if len(full_subpath_parts) > 2:
+                subpath_with_magic_keyword += "/" + "/".join(full_subpath_parts[2:])
+            query_policies.append(
+                f"{space_name}:{subpath_with_magic_keyword.strip('/')}:{resource_type}:{str(is_active).lower()}"
+            )
+
+        if full_subpath == "/":
+            full_subpath = ""
+        else:
+            full_subpath += "/"
+
+    return query_policies
+
+
+def matches_subpath(perm_key: str, space_name: str, query_subpath: str) -> bool:
+    if not perm_key.startswith(f"{space_name}:"):
+        return False
+
+    parts = perm_key.split(":", 2)
+    if len(parts) < 3:
+        return False
+
+    perm_subpath = parts[1].lstrip("/")
+    return (
+        query_subpath == perm_subpath
+        or query_subpath.startswith(perm_subpath + "/")
+    )
+
+
+async def get_user_query_policies(
+    db,
+    user_shortname: str,
+    space_name: str,
+    subpath: str,
+    is_space: bool = False,
+) -> list:
+    """
+    Generate list of query policies based on user's permissions
+    ex: [
+        "products:offers:content:true:admin_shortname", # IF conditions = {"is_active", "own"}
+        "products:offers:content:true:*", # IF conditions = {"is_active"}
+        "products:offers:content:false:admin_shortname|products:offers:content:true:admin_shortname",
+        # ^^^ IF conditions = {"own"}
+        "products:offers:content:*", # IF conditions = {}
+    ]
+    """
+    user_permissions = await db.get_user_permissions(user_shortname)
+    user_groups = (await db.load_user_meta(user_shortname)).groups or []
+    user_groups.append(user_shortname)
+
+    query_subpath = subpath.lstrip("/")
+
+    filtered_permissions = {
+        perm_key: permission
+        for perm_key, permission in user_permissions.items()
+        if 'query' in permission.get('allowed_actions', [])
+        and (
+            is_space
+            or perm_key.startswith(settings.all_spaces_mw)
+            or perm_key.startswith(f'{space_name}:__all_subpaths__')
+            or matches_subpath(perm_key, space_name, query_subpath)
+        )
+    }
+    sql_query_policies = []
+    for perm_key, permission in filtered_permissions.items():
+        perm_key = perm_key.replace(settings.all_spaces_mw, space_name)
+        perm_key = perm_key.replace(settings.all_subpaths_mw, subpath.strip("/"))
+        perm_key = perm_key.strip("/")
+        if (
+                ConditionType.is_active in permission["conditions"]
+                and ConditionType.own in permission["conditions"]
+        ):
+            for user_group in user_groups:
+                sql_query_policies.append(f"{perm_key}:true:{user_group}")
+        elif ConditionType.is_active in permission["conditions"]:
+            sql_query_policies.append(f"{perm_key}:true:*")
+        elif ConditionType.own in permission["conditions"]:
+            for user_group in user_groups:
+                if settings.active_data_db == 'file':
+                    sql_query_policies.append(
+                        f"{perm_key}:true:{user_shortname}|{perm_key}:false:{user_group}"
+                    )
+                else:
+                    sql_query_policies.append(f"{perm_key}:true:{user_shortname}")
+                    sql_query_policies.append(f"{perm_key}:false:{user_shortname}")
+
+        else:
+            sql_query_policies.append(f"{perm_key}:*")
+    return sql_query_policies