dmart 1.4.40.post8__py3-none-any.whl

dmart/test_utils.py

@@ -0,0 +1,34 @@
+from fastapi import status
+
+
+def check_repeated_shortname(response):
+    json_response = response.json()
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    assert "failed" == json_response.get("status")
+    assert "request" == json_response.get("error", {}).get("type")
+
+
+def check_not_found(response):
+    json_response = response.json()
+    assert response.status_code == status.HTTP_404_NOT_FOUND
+    assert "failed" == json_response.get("status")
+    assert "db" == json_response.get("error").get("type")
+
+
+def check_unauthorized(response):
+    json_response = response.json()
+    assert response.status_code == status.HTTP_401_UNAUTHORIZED
+    assert "failed" == json_response.get("status")
+    assert "auth" == json_response.get("error", {}).get("type")
+
+
+def assert_code_and_status_success(response):
+    if response.status_code != status.HTTP_200_OK:
+        print(
+            "\n\n\n\n\n========================= ERROR RESPONSE: =========================n:",
+            response.json(),
+            "\n\n\n\n\n",
+        )
+    assert response.status_code == status.HTTP_200_OK
+    json_response = response.json()
+    assert json_response["status"] == "success"

dmart/utils/access_control.py

@@ -0,0 +1,306 @@
+import re
+import sys
+from models.core import Meta, ACL, ActionType, ConditionType, Group, Permission, Role, User
+from models.enums import ResourceType
+from utils.helpers import camel_case, flatten_dict
+from utils.settings import settings
+from data_adapters.adapter import data_adapter as db
+from utils.regex import FILE_PATTERN
+
+
+class AccessControl:
+    permissions: dict[str, Permission] = {}
+    groups: dict[str, Group] = {}
+    roles: dict[str, Role] = {}
+    users: dict[str, User] = {}
+
+    async def load_permissions_and_roles(self) -> None:
+        if settings.active_data_db == "file":
+            management_path = settings.spaces_folder / settings.management_space
+
+            management_modules: dict[str, type[Meta]] = {
+                "groups": Group,
+                "roles": Role,
+                "permissions": Permission
+            }
+
+            for module_name, module_value in management_modules.items():
+                self_module = getattr(self, module_name)
+                self_module = {}
+                path = management_path / module_name
+                entries_glob = ".dm/*/meta.*.json"
+                if path.exists():
+                    for one in path.glob(entries_glob):
+                        match = FILE_PATTERN.search(str(one))
+                        if not match or not one.is_file():
+                            continue
+                        shortname = match.group(1)
+                        try:
+                            resource_obj: Meta = await db.load(
+                                settings.management_space,
+                                module_name,
+                                shortname,
+                                module_value,
+                                "anonymous",
+                            )
+                            if resource_obj.is_active:
+                                self_module[shortname] = resource_obj  # store in redis doc
+                        except Exception as ex:
+                            # print(f"Error processing @{settings.management_space}/{module_name}/{shortname} ... ", ex)
+                            raise ex
+
+            await db.create_user_premission_index()
+            await db.store_modules_to_redis(self.roles, self.groups, self.permissions)
+            await db.delete_user_permissions_map_in_redis()
+
+
+    async def check_access(
+            self,
+            user_shortname: str,
+            space_name: str,
+            subpath: str,
+            resource_type: ResourceType,
+            action_type: ActionType,
+            resource_is_active: bool = False,
+            resource_owner_shortname: str | None = None,
+            resource_owner_group: str | None = None,
+            record_attributes: dict = {},
+            entry_shortname: str | None = None
+    ):
+        # print("Checking access for", user_shortname, space_name, subpath, resource_type, action_type)
+        if resource_type == ResourceType.space and entry_shortname:
+            return await self.check_space_access(
+                user_shortname,
+                entry_shortname
+            )
+        
+        if entry_shortname:
+            acl_access = await self.check_access_control_list(
+                space_name,
+                subpath,
+                resource_type,
+                entry_shortname,
+                action_type,
+                user_shortname
+            )
+            if acl_access:
+                return True
+        # print("Checking check_space_access access")
+        user_permissions = await db.get_user_permissions(user_shortname)
+
+        user_groups = (await db.load_user_meta(user_shortname)).groups or []
+
+        # Generate set of achevied conditions on the resource
+        # ex: {"is_active", "own"}
+        resource_achieved_conditions: set[ConditionType] = set()
+        if resource_is_active:
+            resource_achieved_conditions.add(ConditionType.is_active)
+        if resource_owner_shortname == user_shortname or resource_owner_group in user_groups:
+            resource_achieved_conditions.add(ConditionType.own)
+
+        # Allow checking for root permissions
+        subpath_parts = ["/"]
+        subpath_parts += list(filter(None, subpath.strip("/").split("/")))
+        if resource_type == ResourceType.folder and entry_shortname:
+            subpath_parts.append(entry_shortname)
+
+        search_subpath = ""
+        for subpath_part in subpath_parts:
+            search_subpath += subpath_part
+            # Check if the user has global access
+            global_access = self.has_global_access(
+                space_name,
+                user_permissions,
+                search_subpath,
+                action_type,
+                resource_type,
+                resource_achieved_conditions,
+                record_attributes
+            )
+            if global_access:
+                return True
+
+            permission_key = f"{space_name}:{search_subpath}:{resource_type}"
+            if (
+                permission_key in user_permissions
+                and action_type in user_permissions[permission_key]["allowed_actions"]
+                and self.check_access_conditions(
+                    set(user_permissions[permission_key]["conditions"]),
+                    set(resource_achieved_conditions),
+                    action_type,
+                )
+                and self.check_access_restriction(
+                    user_permissions[permission_key]["restricted_fields"],
+                    user_permissions[permission_key]["allowed_fields_values"],
+                    action_type,
+                    record_attributes
+                )
+            ):
+                return True
+
+            if search_subpath == "/":
+                search_subpath = ""
+            else:
+                search_subpath += "/"
+
+        return False
+
+    async def check_access_control_list(
+            self,
+            space_name: str,
+            subpath: str,
+            resource_type: ResourceType,
+            entry_shortname: str,
+            action_type: ActionType,
+            user_shortname: str,
+    ) -> bool:
+        resource_cls = getattr(
+            sys.modules["models.core"], camel_case(resource_type)
+        )
+        
+        try:
+            entry = await db.load(
+                space_name=space_name,
+                subpath=subpath,
+                shortname=entry_shortname,
+                class_type=resource_cls
+            )
+        except Exception:
+            return False
+            
+        if not entry.acl:
+            return False
+
+        user_acl: ACL | None = None
+        for access in entry.acl:
+            if access.user_shortname == user_shortname:
+                user_acl = access
+                break
+
+        if not user_acl:
+            return False
+
+        return action_type in user_acl.allowed_actions
+
+    def has_global_access(
+            self,
+            space_name: str,
+            user_permissions: dict,
+            search_subpath: str,
+            action_type: ActionType,
+            resource_type: str,
+            resource_achieved_conditions: set,
+            record_attributes: dict
+    ) -> bool:
+        """
+        check if has access to global subpath by replacing the following
+        subpath = / => __all_subpaths__
+        subpath = {subpath} => __all_subpaths__
+        subpath = {subpath}/protected => __all_subpaths__/protected
+        subpath = {subpath}/protected/mine => {subpath}/__all_subpaths__/mine
+        """
+        original_subpath = search_subpath
+        search_subpath_parts = search_subpath.split("/")
+        if len(search_subpath_parts) > 1:
+            search_subpath_parts[-2] = settings.all_subpaths_mw
+            search_subpath = "/".join(search_subpath_parts)
+        elif len(search_subpath_parts) == 1:
+            search_subpath = settings.all_subpaths_mw
+        if search_subpath[-1] == "/" and len(search_subpath) > 1:
+            search_subpath = search_subpath[:-1]
+
+        permission_key = None
+        # check if has access to all spaces
+        if f"{settings.all_spaces_mw}:{search_subpath}:{resource_type}" in user_permissions:
+            permission_key = f"{settings.all_spaces_mw}:{search_subpath}:{resource_type}"
+
+        # check if has access to current spaces
+        if f"{space_name}:{search_subpath}:{resource_type}" in user_permissions:
+            permission_key = f"{space_name}:{search_subpath}:{resource_type}"
+
+
+        # check if has access to current subpath
+        if f"{settings.all_spaces_mw}:{original_subpath}:{resource_type}" in user_permissions:
+            permission_key = f"{settings.all_spaces_mw}:{original_subpath}:{resource_type}"
+
+        if not permission_key:
+            return False
+
+        if (
+                action_type in user_permissions[permission_key]["allowed_actions"]
+                and self.check_access_conditions(
+            set(user_permissions[permission_key]["conditions"]),
+            set(resource_achieved_conditions),
+            action_type,
+        )
+                and self.check_access_restriction(
+            user_permissions[permission_key]["restricted_fields"],
+            user_permissions[permission_key]["allowed_fields_values"],
+            action_type,
+            record_attributes
+        )
+        ):
+            return True
+
+        return False
+
+    def check_access_conditions(
+            self,
+            premission_conditions: set,
+            resource_achieved_conditions: set,
+            action_type: ActionType,
+    ):
+        # actions of type query will be handled in the query function
+        # actions of type create shouldn't check for permission conditions
+        if action_type in [ActionType.create, ActionType.query]:
+            return True
+
+        return premission_conditions.issubset(resource_achieved_conditions)
+
+    def check_access_restriction(
+            self,
+            restricted_fields: list,
+            allowed_fields_values: dict,
+            action_type: ActionType,
+            record_attributes: dict
+    ):
+        """
+        in case of create or update action, check access for the record fields
+        via permission.restricted_fields and permission.allowed_fields_values
+        """
+        if action_type not in [ActionType.create, ActionType.update]:
+            return True
+
+        flattened_attributes = flatten_dict(record_attributes)
+
+        for restricted_field in restricted_fields:
+            if restricted_field in flattened_attributes:
+                return False
+            for flattened_key in flattened_attributes.keys():
+                if flattened_key == restricted_field or flattened_key.startswith(f"{restricted_field}."):
+                    return False
+
+        for field_name, field_values in allowed_fields_values.items():
+            if field_name not in flattened_attributes:
+                continue
+            if (
+                    isinstance(flattened_attributes[field_name], list) and
+                    isinstance(field_values[0], list) and
+                    not any(all(i in allowed_values for i in flattened_attributes[field_name]) for allowed_values in field_values)
+            ):
+                return False
+            elif (
+                    not isinstance(flattened_attributes[field_name], list) and
+                    flattened_attributes[field_name] not in field_values
+            ):
+                return False
+
+        return True
+
+    async def check_space_access(self, user_shortname: str, space_name: str) -> bool:
+        user_permissions = await db.get_user_permissions(user_shortname)
+        prog = re.compile(f"{space_name}:*|{settings.all_spaces_mw}:*")
+        return bool(list(filter(prog.match, user_permissions.keys())))
+
+
+access_control = AccessControl()

dmart/utils/async_request.py

@@ -0,0 +1,8 @@
+import aiohttp
+#import json_logging
+
+
+def AsyncRequest(headers=None):
+    # corr_id = {"X-Correlation-ID": json_logging.get_correlation_id()}
+    # headers = {**headers, **corr_id} if headers else corr_id
+    return aiohttp.ClientSession(headers=headers)

dmart/utils/exporter.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+from hashlib import blake2b, md5
+import json
+import os
+import shutil
+import sys
+import copy
+import jsonschema
+from aiofiles import open as aopen
+from pathlib import Path
+
+# from pydantic import config
+
+
+def hashing_data(data: str, hashed_data: dict):
+    hash = blake2b(salt=md5(data.encode()).digest())
+    hash.update(data.encode())
+    hashed_val = md5(hash.digest()).hexdigest()
+    hashed_data[hashed_val] = data
+    return hashed_val
+
+
+def exit_with_error(msg: str):
+    print("ERROR!!", msg)
+    sys.exit(1)
+
+
+SECURED_FIELDS = [
+    "name",
+    "email",
+    "ip",
+    "pin",
+    "RechargeNumber",
+    "CallingNumber",
+    "shortname",
+    "contact_number",
+    "pin",
+    "msisdn",
+    "imsi",
+    "sender_msisdn",
+    "old_username",
+    "firstname",
+    "lastname",
+]
+OUTPUT_FOLDER_NAME = "spaces_data"
+
+def meta_path(
+    space_path: Path, 
+    subpath: str, 
+    file_path: str, 
+    resource_type: str
+) -> Path:
+    return space_path / f"{subpath}/.dm/{file_path}/meta.{resource_type}.json"    
+
+async def get_meta(
+    *, 
+    space_path: Path, 
+    subpath: str, 
+    file_path: str, 
+    resource_type: str
+):
+    meta_content = meta_path(space_path, subpath, file_path, resource_type)
+    async with aopen(meta_content, "r") as f:
+        return json.loads(await f.read())
+
+
+def validate_config(config_obj: dict):
+    if (
+        not config_obj.get("space")
+        or not config_obj.get("subpath")
+        or not config_obj.get("resource_type")
+        or not config_obj.get("schema_shortname")
+    ):
+        return False
+    return True
+
+
+def remove_fields(src: dict, restricted_keys: list):
+    for k in list(src.keys()):
+        if isinstance(src[k], list):
+            for item in src[k]:
+                if isinstance(item, dict):
+                    item = remove_fields(item, restricted_keys)
+        elif isinstance(src[k], dict):
+            src[k] = remove_fields(src[k], restricted_keys)
+            
+        if k in restricted_keys:
+            del src[k]
+
+    return src
+
+def enc_dict(d: dict, hashed_data: dict):
+    for k, v in d.items():
+        if isinstance(v, dict):
+            d[k] = enc_dict(v, hashed_data)
+        elif isinstance(d[k], list):
+            for item in d[k]:
+                if isinstance(item, dict):
+                    item = enc_dict(item, hashed_data)
+
+        # if k == "msisdn":
+        #     d[k] = hashing_data(str(v))
+        #     try:
+        #         print("ADD TO SQLLITE")
+        #         cur.execute(f"INSERT INTO migrated_data VALUES('{v}', '{d[k]}')")
+        #         con.commit()
+        #         print("FINSIHED")
+        #     except sqlite3.IntegrityError as e:
+        #         if "migrated_data.hash" in str(e):  # msisdn already in db
+        #             raise Exception(f"Collision on: msisdn {v}, hash {d[k]}")
+        elif k in SECURED_FIELDS:
+            d[k] = hashing_data(str(v), hashed_data)
+
+    return d
+
+
+def prepare_output(
+    meta: dict,
+    payload: dict,
+    included_meta_fields: dict,
+    excluded_payload_fields: dict,
+):
+    out = payload
+    for field_meta in included_meta_fields:
+        field_name = field_meta.get("field_name")
+        rename_to = field_meta.get("rename_to")
+        if not field_name:
+            continue
+        if rename_to:
+            out[rename_to] = meta.get(field_name, "")
+        else:
+            out[field_name] = meta.get(field_name, "")
+
+    out = remove_fields(
+        out, 
+        [field["field_name"] for field in excluded_payload_fields]
+    )
+    return out
+
+
+                
+
+async def extract(
+        space : str, 
+        subpath : str, # = config_obj.get("subpath")
+        resource_type : str, #  = config_obj.get("resource_type")
+        schema_shortname : str, #  = config_obj.get("schema_shortname")
+        included_meta_fields : dict, #  = config_obj.get("included_meta_fields", [])
+        excluded_payload_fields : dict, # = config_obj.get("excluded_payload_fields", [])
+        spaces_path: str, 
+        output_path : str, 
+    entries_since = None
+) -> None:
+    hashed_data: dict[str, str] = {}
+
+    space_path = Path(f"{spaces_path}/{space}")
+    subpath_schema_obj = None
+    with open(space_path / f"schema/{schema_shortname}.json", "r") as f:
+        subpath_schema_obj = json.load(f)
+    input_subpath_schema_obj = copy.deepcopy(subpath_schema_obj)
+
+    output_subpath = Path(f"{output_path}/{OUTPUT_FOLDER_NAME}/{space}/{subpath}")
+    if not output_subpath.is_dir():
+        os.makedirs(output_subpath)
+
+    # Generat output schema
+    schema_fil = output_subpath / "schema.json"
+    for field in included_meta_fields:
+        if "oneOf" in subpath_schema_obj:
+            for schema in subpath_schema_obj["oneOf"]:
+                schema["properties"][field["field_name"]] = field["schema_entry"]
+                if field.get("rename_to"):
+                    schema["properties"][field["rename_to"]] = schema[
+                        "properties"
+                    ].pop(field["field_name"])
+        else:
+            subpath_schema_obj["properties"][field["field_name"]] = field["schema_entry"]
+            if field.get("rename_to"):
+                subpath_schema_obj["properties"][field["rename_to"]] = subpath_schema_obj[
+                    "properties"
+                ].pop(field["field_name"])
+    if "oneOf" in subpath_schema_obj:
+        for schema in subpath_schema_obj["oneOf"]:
+            schema["properties"] = remove_fields(
+                schema["properties"], 
+                [field["field_name"] for field in excluded_payload_fields]
+            )
+    else:
+        subpath_schema_obj["properties"] = remove_fields(
+            subpath_schema_obj["properties"], 
+            [field["field_name"] for field in excluded_payload_fields]
+        )
+    open(schema_fil, "w").write(json.dumps(subpath_schema_obj) + "\n")
+
+    # Generat output content file
+    data_file = output_subpath / "data.ljson"
+    path = os.path.join(spaces_path, space, subpath)
+    for file_name in os.listdir(path):
+        if not file_name.endswith(".json"):
+            continue
+        if entries_since:
+            payload_ts = int(round(
+                os.path.getmtime(os.path.join(path, file_name)) * 1000
+            ))
+            meta_ts = int(round(os.path.getmtime(meta_path(
+                space_path, 
+                subpath, 
+                file_name.split(".")[0], 
+                resource_type
+            )) * 1000))
+            if payload_ts <= entries_since and meta_ts <= entries_since:
+                continue
+            
+        async with aopen(os.path.join(path, file_name), "r") as f:
+            content = await f.read()
+        try:
+            payload = json.loads(content)
+            jsonschema.validate(
+                instance=payload, schema=input_subpath_schema_obj
+            )
+            meta = await get_meta(
+                space_path=space_path,
+                subpath=subpath,
+                file_path=file_name.split(".")[0],
+                resource_type=resource_type,
+            )
+        except Exception:
+            # print(f"ERROR: {error.args = }")
+            continue
+
+        out = prepare_output(
+            meta, payload, included_meta_fields, excluded_payload_fields
+        )
+
+        # jsonschema.validate(instance=out, schema=subpath_schema_obj)
+
+        encrypted_out = enc_dict(out, hashed_data)
+        open(data_file, "a").write(json.dumps(encrypted_out) + "\n")
+        
+    open(f"{output_subpath}/hashed_data.json", "w").write(json.dumps(hashed_data))
+        
+
+
+async def main(tasks):
+    await asyncio.gather(*tasks)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config", required=True, help="Json config relative path from the script"
+    )
+    parser.add_argument(
+        "--spaces", required=True, help="Spaces relative path from the script"
+    )
+    parser.add_argument(
+        "--output",
+        help="Output relative path from the script (the default path is the current script path",
+    )
+    parser.add_argument(
+        "--since",
+        help="Export entries created/updated since the provided timestamp",
+    )
+    args = parser.parse_args()
+    since = None
+    output_path = ""
+    if args.output:
+        output_path = args.output
+        
+    if args.since:
+        since = int(round(float(args.since) * 1000))
+
+    if not os.path.isdir(args.spaces):
+        exit_with_error(f"The spaces folder {args.spaces} is not found.")
+
+    out_path = os.path.join(output_path, OUTPUT_FOLDER_NAME)
+    if os.path.isdir(out_path):
+        shutil.rmtree(out_path)
+
+    # con = sqlite3.connect(f"../../exporter_data/output/data.db")
+    # cur = con.cursor()
+    # cur.execute("DROP TABLE IF EXISTS migrated_data")
+    # cur.execute(
+    #     "CREATE TABLE migrated_data(hash VARCHAR UNIQUE, msisdn VARCHAR UNIQUE)"
+    # )
+    # print(con)
+
+    tasks = []
+    with open(args.config, "r") as f:
+        config_objs = json.load(f)
+
+    for config_obj in config_objs:
+        if not validate_config(config_obj):
+            continue
+        tasks.append(extract(config_obj.get("space", ""), 
+                             config_obj.get("subpath", ""), 
+                             config_obj.get("resource_type", ""), 
+                             config_obj.get("schema_shortname", ""), 
+                             config_obj.get("included_meta_fields", {}), 
+                             config_obj.get("excluded_payload_fields", {}), 
+                             args.spaces, output_path, since))
+
+    asyncio.run(main(tasks))
+
+    print(
+        f"Output path: {os.path.abspath(os.path.join(output_path, OUTPUT_FOLDER_NAME))}"
+    )