dmart 1.4.17__py3-none-any.whl

schema_migration.py

@@ -0,0 +1,208 @@
+import argparse
+import asyncio
+from enum import Enum
+import re
+import sys
+from models.enums import ContentType
+from utils import helpers
+from data_adapters.adapter import data_adapter as db
+from models.core import Meta, Schema
+from utils.settings import settings
+
+
+class FieldType(Enum):
+    string = "string"
+    number = "number"
+    integer = "integer"
+    array = "array"
+    
+    
+FIELD_TYPE_PARSER: dict = {
+    "string": str,
+    "number": int,
+    "integer": int,
+    "array": list
+}
+    
+    
+async def change_field_type(
+    space: str,
+    schema_model: Schema,
+    schema_payload: dict,
+    field: str,
+    new_type: FieldType
+):
+    # 3-update field type to new_type
+    schema_properties = schema_payload["properties"]
+    await db.internal_sys_update_model(
+        space_name=space,
+        subpath="schema",
+        meta=schema_model,
+        payload_dict=schema_payload,
+        updates={
+            "properties": schema_properties
+        },
+    )
+    
+    updated_num = 0
+    
+    # 4-loop over space by glob "*/.dm/*/meta.*.json"
+    path = settings.spaces_folder / space
+    entries_glob = "*/.dm/*/meta.*.json"
+    FILE_PATTERN = re.compile("(\\w*)\\/\\.dm\\/(\\w*)\\/meta\\.([a-zA-Z]*)\\.json$")
+    for one in path.glob(entries_glob):
+        match = FILE_PATTERN.search(str(one))
+        if not match or not one.is_file():
+            continue
+        subpath = match.group(1)
+        shortname = match.group(2)
+        resource_type = match.group(3)
+        resource_cls = getattr(
+            sys.modules["models.core"], helpers.camel_case(resource_type)
+        )
+        try:
+            resource_obj: Meta = await db.load(
+                space_name=space,
+                subpath=subpath,
+                shortname=shortname,
+                class_type=resource_cls
+            )
+            
+            # 5-if resource.schema_shortname == schema:
+            if(
+                not resource_obj.payload or
+                not isinstance(resource_obj.payload.body, str) or
+                resource_obj.payload.schema_shortname != schema_model.shortname
+            ):
+                continue
+            
+            # 5.1-load payload file
+            resource_payload = await db.load_resource_payload(
+                space_name=space,
+                subpath=subpath,
+                filename=resource_obj.payload.body,
+                class_type=resource_cls
+            )
+        except Exception as ex:
+            print(f"Error loading {one}", ex)
+            continue
+        
+        resource_payload_keys = helpers.flatten_dict(resource_payload)
+        if field not in resource_payload_keys or not resource_payload:
+            continue
+        
+        # 5.2-parse field's old_type to new_type
+        field_tree = field.split(".")
+        last_idx = len(field_tree)-1
+        main_field = resource_payload[field_tree[0]]
+        field_to_update = main_field
+        for i in range(1, last_idx):
+            field_to_update = main_field[field_tree[i]] #type: ignore
+        field_to_update[field_tree[last_idx]] = FIELD_TYPE_PARSER[new_type](field_to_update[field_tree[last_idx]])#type: ignore
+        await db.internal_sys_update_model(
+            space_name=space,
+            subpath=subpath,
+            meta=resource_obj,
+            payload_dict=resource_payload,
+            updates={
+                field_tree[0]: main_field
+            },
+        )
+        updated_num += 1
+        
+    return updated_num
+    
+
+async def main(
+    space: str,
+    schema: str,
+    field: str,
+    old_type: FieldType,
+    new_type: FieldType
+):
+    """
+    Algorithm:
+    1-load schema
+    2-make sure field with old_type exist
+    3-update field type to new_type
+    4-loop over space by glob "*/.dm/*/meta.*.json"
+    5-if resource.schema_shortname == schema:
+        5.1-load payload file
+        5.2-parse field's old_type to new_type
+    """
+    
+    # 1-load schema
+    schema_model: Schema = Schema.model_validate(
+        await db.load(
+            space_name=space,
+            subpath="schema",
+            shortname=schema,
+            class_type=Schema
+        )
+    )
+    if(
+        not schema_model.payload or
+        schema_model.payload.content_type != ContentType.json or
+        not isinstance(schema_model.payload.body, str)
+    ):
+        print(f"Invalid schema file: \n{schema_model.model_dump_json()}")
+        return
+    schema_payload = await db.load_resource_payload(
+        space_name=space,
+        subpath="schema",
+        filename=schema_model.payload.body,
+        class_type=Schema
+    )
+    
+    if not schema_payload: 
+        return
+    
+    # 2-make sure field with old_type exist
+    field_tree = field.split(".")
+    field_definition = schema_payload
+    for f in field_tree:
+        field_definition = field_definition["properties"].get(f, {})
+        
+    if field_definition.get("type") != old_type:
+        print("Invalid old type for the specified field under the specified schema")
+        return
+        
+    field_definition["type"] = new_type
+        
+    updated_num = await change_field_type(
+        space,
+        schema_model,
+        schema_payload,
+        field,
+        new_type
+    )
+    
+    print(f"Successfully updated the schema along with {updated_num} resource files")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Change field type in a specific Schema under a specific Space",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("-p", "--space", help="Space name")
+    parser.add_argument("-c", "--schema", help="Schema name")
+    parser.add_argument("-f", "--field", help="Field name to change")
+    parser.add_argument(
+        "--old-type", 
+        help="Field's old type, supported types: string, number, integer, array"
+    )
+    parser.add_argument(
+        "--new-type", 
+        help="Field's new type, supported types: string, number, integer, array"
+    )
+
+    args = parser.parse_args()
+
+    asyncio.run(main(
+        args.space, 
+        args.schema, 
+        args.field, 
+        args.old_type,
+        args.new_type
+    ))

schema_modulate.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env -S BACKEND_ENV=config.env python3
+import asyncio
+from enum import StrEnum
+
+from sqlalchemy import update
+from typing import Any
+from data_adapters.sql.adapter import SQLAdapter
+from data_adapters.sql.create_tables import Users, Roles, Permissions, Entries, Spaces, Attachments
+from utils.settings import settings
+import argparse
+from sqlmodel import select, col
+
+
+'''
+--space and --subpath are optional
+
+# add new key to the records
+## with default value
+./schema_modulate.py --space management --subpath users -t +payload.body.xxx -v 123
+## default value is None
+./schema_modulate.py --space management --subpath users -t +payload.body.xxx
+
+# remove key from the records
+./schema_modulate.py --space management --subpath users -t ~payload.body.xxx
+
+# update key in the records
+./schema_modulate.py --space management --subpath users -t payload.body.xxx -v yyy
+'''
+
+class ResourceType(StrEnum):
+    add = "add"
+    remove = "remove"
+    update = "update"
+
+
+async def handle_sql_modulation(args):
+    spaces: list[Any] = []
+    if args.space:
+        if args.space == "management":
+            if args.subpath is None:
+                spaces = []
+            if args.subpath == "~attachments":
+                spaces = [Attachments]
+            else:
+                if args.subpath == "users":
+                    spaces = [Users]
+                elif args.subpath == "roles":
+                    spaces = [Roles]
+                elif args.subpath == "permissions":
+                    spaces = [Permissions]
+                elif args.subpath == "~spaces":
+                    spaces = [Permissions]
+                else:
+                    spaces = [Entries]
+        else:
+            spaces = [Entries]
+            if args.subpath == "~attachments":
+                spaces.append(Attachments)
+    else:
+        spaces = [Entries, Users, Roles, Permissions, Spaces, Attachments]
+
+    targets = args.target.split(".")
+
+    state = ResourceType.update
+    if targets[0].startswith("+"):
+        print(f"[info] Adding new key '{targets[0]}' to the records")
+        state = ResourceType.add
+        targets[0] = targets[0][1:]
+
+        if args.value:
+            print("[warn] flag -v --value is not required for adding new key")
+    if targets[0].startswith("~"):
+        print(f"[info] Removing key '{targets[0]}' from the records")
+        state = ResourceType.remove
+        targets[0] = targets[0][1:]
+
+        if args.value:
+            print("[warn] flag -v --value is not required for removing key")
+    else:
+        print(f"[info] Altering the key '{targets[0]}' fo records")\
+
+    if targets[0] not in ["description", "displayname", "payload"]:
+        raise Exception("target must be either 'description', 'displayname' or 'payload'")
+
+    async with SQLAdapter().get_session() as session:
+        for space in spaces:
+            print("[info] Processing...", space)
+
+            statement = select(space)
+
+            if space not in [Users, Roles, Permissions, Spaces, Attachments]:
+                statement = statement.where(space.shortname == args.space)
+                if args.subpath:
+                    statement = statement.where(space.subpath == args.subpath)
+
+            records = (await session.execute(statement)).all()
+            records = [record[0] for record in records]
+            print("[info] # Records found:", len(records))
+            print("[info] state:", state)
+            for record in records:
+                if hasattr(record, targets[0]):
+                    if state == ResourceType.update:
+                        obj = getattr(record, targets[0])
+                        if obj:
+                            keys = targets
+                            sub_obj = obj
+                            for key in keys[1:-1]:
+                                if not bool(sub_obj):
+                                    break
+                                sub_obj = sub_obj.get(key, {})
+
+                            if not bool(sub_obj):
+                                continue
+
+                            if keys[-1] in sub_obj:
+                                sub_obj[args.value] = sub_obj.pop(keys[-1])
+
+                    elif state == ResourceType.add:
+                        obj = getattr(record, targets[0])
+                        if obj is None:
+                            setattr(record, targets[0], {})
+                        keys = targets
+                        sub_obj = obj
+                        for key in keys[1:-1]:
+                            if not bool(sub_obj):
+                                break
+                            sub_obj = sub_obj.get(key, {})
+
+                        if not bool(sub_obj):
+                            continue
+
+                        if keys[-1] not in sub_obj:
+                            sub_obj[keys[-1]] = args.value
+
+                    elif state == ResourceType.remove:
+                        obj = getattr(record, targets[0])
+                        if obj:
+                            keys = targets
+                            sub_obj = obj
+                            for key in keys[1:-1]:
+                                sub_obj = sub_obj.get(key, {})
+                            if not bool(sub_obj):
+                                continue
+                            if keys[-1] in sub_obj:
+                                sub_obj.pop(keys[-1])
+
+                            print(obj)
+                            setattr(record, targets[0], obj)
+
+                stmt = update(space).where(col(space.uuid )== record.uuid).values(
+                    **{targets[0]: getattr(record, targets[0])})
+                await session.execute(stmt)  #type: ignore
+            await session.commit()
+
+
+
+def handle_file_modulation(args):
+    pass
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Modulate schema field type",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--space",
+        required=True
+    )
+    parser.add_argument(
+        "--subpath",
+        default=None
+    )
+    parser.add_argument(
+        "-t", "--target",
+        required=True
+    )
+    parser.add_argument(
+        "-v", "--value",
+        default=None
+    )
+
+    args = parser.parse_args()
+
+    if settings.active_data_db == "sql":
+        asyncio.run(
+            handle_sql_modulation(args)
+        )
+    else:
+        handle_file_modulation(args)

set_admin_passwd.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env -S BACKEND_ENV=config.env python3
+import asyncio
+import json
+import getpass
+import subprocess
+import utils.regex as regex
+import re
+from pathlib import Path
+from sqlmodel import select
+
+from data_adapters.sql.adapter import SQLAdapter
+from data_adapters.sql.create_tables import Users
+from utils.password_hashing import hash_password
+from utils.settings import settings
+
+
+users : dict[str, dict]= {"dmart":{}, "alibaba": {}}
+
+while True:
+    password = getpass.getpass("Type the new password for admin/testuser then hit enter: ")
+    if re.match(regex.PASSWORD, password):
+        break
+    else:
+        print("Password didn't match the rules: >= 8 chars that are Alphanumeric mix cap/small with _#@%*!?$^- ")
+
+print("Generating and storing the password for dmart and alibaba")
+hashed = hash_password(password)
+
+async def main():
+    if settings.active_data_db == "file":
+        for key in users.keys():
+            file_name = settings.spaces_folder / f"management/users/.dm/{key}/meta.user.json"
+            with open(file_name, 'r') as read_file:
+                data = json.load(read_file)
+                data["password"] = hashed
+                with open(file_name, 'w') as write_file:
+                    write_file.write(json.dumps(data))
+    else:
+        async with SQLAdapter().get_session() as session:
+            for key in users.keys():
+                statement = select(Users).where(Users.shortname == key)
+                user = (await session.execute(statement)).one()[0]
+                user.password=hashed
+                user.is_active=True
+                session.add(user)
+                await session.commit()
+
+
+asyncio.run(main())
+
+login_creds_sample = Path(__file__).resolve().parent / "login_creds.sh.sample"
+if login_creds_sample.exists():
+    with open("./login_creds.sh", 'w') as creds:
+        subprocess.run( [ "sed", f"s/xxxx/{password}/g", str(login_creds_sample) ], stdout=creds)
+print("Done")

sync.py

@@ -0,0 +1,202 @@
+import argparse
+import hashlib
+import json
+import requests
+
+from models.enums import RequestType
+from utils.settings import settings
+
+
+local_username = "dmart"
+local_password = "Test1234"
+
+
+headers = {
+    "accept": "application/json, text/plain, */*",
+}
+target_headers = {
+    **headers,
+}
+local_headers = {
+     **headers,
+}
+
+body = {
+    "filter_shortnames": [],
+    "type": "search",
+    "exact_subpath": True,
+    "limit": 100,
+    "offset": 0,
+    "search": "",
+    "retrieve_json_payload": True,
+}
+
+
+def login(username, password, target):
+    body = {
+        "shortname": username,
+        "password": password,
+    }
+
+    response = requests.post(f"{target}/user/login", headers=headers, json=body,)
+
+    if response.ok:
+        return response.json()["records"][0]["attributes"]["access_token"]
+    else:
+        print(f"Error: {response.status_code}, {response.text}")
+
+
+def hash_records(local_records, target_records):
+    hashed_local_records = {}
+    hashed_target_records = {}
+    for local_record in local_records:
+        sha1 = hashlib.sha1()
+        r = {
+            "shortname": local_record["shortname"],
+            "displayname": local_record.get("attributes", {}).get("displayname", {}),
+            "description": local_record.get("attributes", {}).get("description", {}),
+        }
+        if local_record.get("attributes", {}).get("payload", {}):
+            r["payload"] = local_record.get("attributes", {}).get("payload", {}).get("checksum", {})
+        else:
+            r["payload"] = None
+        sha1.update(json.dumps(r).encode())
+        checksum = sha1.hexdigest()
+        hashed_local_records[local_record["shortname"]] = checksum
+
+    for target_record in target_records:
+        sha1 = hashlib.sha1()
+        r = {
+            "shortname": target_record["shortname"],
+            "displayname": target_record.get("attributes", {}).get("displayname", {}),
+            "description": target_record.get("attributes", {}).get("description", {}),
+        }
+        if target_record.get("attributes", {}).get("payload", {}):
+            r["payload"] = target_record.get("attributes", {}).get("payload", {}).get("checksum", {})
+        else:
+            r["payload"] = None
+        sha1.update(json.dumps(r).encode())
+        checksum = sha1.hexdigest()
+        hashed_target_records[target_record["shortname"]] = checksum
+
+    return hashed_local_records, hashed_target_records
+
+
+local_records = []
+target_records = []
+def fetch_locators(space, subpath, target):
+    global local_records
+    global target_records
+
+    body["space_name"] = space
+    body["subpath"] = subpath
+
+    response_target = requests.post(f"{target}/managed/query", headers=target_headers, json=body,)
+    response_local = requests.post(f"http://{settings.listening_host}:{settings.listening_port}/managed/query", headers=local_headers, json=body,)
+
+    if response_target.ok:
+        target_records = response_target.json()["records"]
+        print('# target records:', len(target_records))
+    else:
+        print(f"Error: {response_target.status_code}, {response_target.text}")
+
+    if response_local.ok:
+        local_records = response_local.json()["records"]
+        print('# local records:', len(local_records))
+    else:
+        print(f"Error: {response_local.status_code}, {response_local.text}")
+
+    return local_records, target_records
+
+
+def get_diff(hashed_local_records, hashed_target_records):
+    added_records = {k: v for k, v in hashed_local_records.items() if k not in hashed_target_records}
+    removed_records = {k: v for k, v in hashed_target_records.items() if k not in hashed_local_records}
+    different_records = {k: v for k, v in hashed_local_records.items() if k in hashed_target_records and hashed_target_records[k] != v}
+
+    print(f"Added records: {added_records}")
+    print(f"Removed records: {removed_records}")
+    print(f"Different records: {different_records}")
+    return added_records, removed_records, different_records
+
+
+def apply_changes(space, target, added_records, removed_records, different_records):
+    added_records_shortnames = [record for record in local_records if record["shortname"] in added_records]
+    removed_records_shortnames = [record for record in target_records if record["shortname"] in removed_records]
+    different_records_shortnames = [record for record in local_records if record["shortname"] in different_records]
+
+    print(f"Added records shortnames: {added_records_shortnames}")
+    print(f"Removed records shortnames: {removed_records_shortnames}")
+    print(f"Different records shortnames: {different_records_shortnames}")
+
+    request_data = {
+        "space_name": space,
+        "request_type": RequestType.create,
+        "records": added_records_shortnames,
+    }
+    response = requests.post(f"{target}/managed/request", headers=target_headers, json=request_data,)
+    if response.ok:
+        print('records:', response.json())
+    else:
+        print(f"Error: {response.status_code}, {response.text}")
+
+    request_data = {
+        "space_name": space,
+        "request_type": RequestType.delete,
+        "records": removed_records_shortnames,
+    }
+    response = requests.post(f"{target}/managed/request", headers=target_headers, json=request_data,)
+    if response.ok:
+        print('records:', response.json())
+    else:
+        print(f"Error: {response.status_code}, {response.text}")
+
+    request_data = {
+        "space_name": space,
+        "request_type": RequestType.update,
+        "records": different_records_shortnames,
+    }
+    response = requests.post(f"{target}/managed/request", headers=target_headers, json=request_data,)
+    if response.ok:
+        print('records:', response.json())
+    else:
+        print(f"Error: {response.status_code}, {response.text}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Process some arguments.")
+    parser.add_argument('-u', required=True, help='The username argument')
+    parser.add_argument('-p', required=True, help='The password argument')
+    parser.add_argument('-sp', required=True, help='The space argument')
+    parser.add_argument('-su', required=True, help='The subpath argument')
+    parser.add_argument('-t', required=True, help='The target argument')
+    parser.add_argument('-l', required=False, help='The limit argument')
+    parser.add_argument('-o', required=False, help='The offset argument')
+
+    args = parser.parse_args()
+
+    print(f">Space: {args.sp}")
+    print(f">Subpath: {args.su}")
+    print(f">Target: {args.t}")
+    print(f">Username: {args.u}")
+
+    if args.l:
+        body["limit"] = args.l
+    if args.o:
+        body["offset"] = args.o
+
+    local_token = login(local_username, local_password, f"http://{settings.listening_host}:{settings.listening_port}")
+    local_headers["Authorization"] = f"Bearer {local_token}"
+
+    target_token = login(args.u, args.p, args.t)
+    target_headers["Authorization"] = f"Bearer {target_token}"
+
+    local_records, target_records = fetch_locators(args.sp, args.su, args.t)
+    hashed_local_records, hashed_target_records = hash_records(local_records, target_records)
+    added_records, removed_records, different_records = get_diff(hashed_local_records, hashed_target_records)
+    apply_changes(args.sp, args.t, added_records, removed_records, different_records)
+
+
+
+if __name__ == "__main__":
+    main()