girder-jsonforms 2.0.0__py3-none-any.whl

girder_jsonforms/__init__.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import json
+import logging
+import os
+
+from girder import events
+from girder.constants import AccessType
+from girder.models.file import File
+from girder.models.item import Item
+from girder.plugin import GirderPlugin, registerPluginStaticContent
+from girder.utility.model_importer import ModelImporter
+
+from .lib.google_drive import authenticate_gdrive, upload_file_to_gdrive
+from .models.entry import FormEntry as FormEntryModel
+from .models.form import Form as FormModel
+from .rest.entry import FormEntry
+from .rest.form import Form
+
+GDRIVE_SERVICE = None
+logger = logging.getLogger(__name__)
+
+
+def annotate_uploads(event):
+    info = event.info
+    if "reference" not in info:
+        return
+
+    try:
+        reference = json.loads(info["reference"])
+    except (ValueError, TypeError):
+        return
+
+    if reference.get("annotate"):
+        parent = Item().load(
+            info["file"]["itemId"], level=AccessType.WRITE, user=info["currentUser"]
+        )
+        reference.pop("file", None)
+        reference.pop("annotate", None)
+        Item().setMetadata(parent, reference)
+
+
+def upload_to_gdrive(event):
+    global GDRIVE_SERVICE
+    info = event.info
+    file = info["file"]
+
+    with File().open(file) as fh:
+        gdrive_file_id = upload_file_to_gdrive(
+            GDRIVE_SERVICE,
+            info["gdriveFolderId"],
+            info["path"],
+            fh,
+            mimetype=file["mimeType"],
+        )
+    parent = Item().load(
+        file["itemId"], level=AccessType.WRITE, user=info["currentUser"]
+    )
+    Item().setMetadata(parent, {"gdriveFileId": gdrive_file_id})
+
+
+class JSONFormsPlugin(GirderPlugin):
+    DISPLAY_NAME = "JSON Forms"
+
+    def load(self, info):
+        ModelImporter.registerModel("form", FormModel, plugin="jsonforms")
+        ModelImporter.registerModel("entry", FormEntryModel, plugin="jsonforms")
+        global GDRIVE_SERVICE
+        try:
+            GDRIVE_SERVICE = authenticate_gdrive()
+        except ValueError:
+            logger.exception("Failed to authenticate with Google Drive")
+        info["apiRoot"].form = Form()
+        info["apiRoot"].entry = FormEntry()
+        events.bind("data.process", "jsonforms", annotate_uploads)
+        events.bind("gdrive.upload", "jsonforms", upload_to_gdrive)
+        registerPluginStaticContent(
+            plugin="jsonforms",
+            css=["/style.css"],
+            js=["/girder-plugin-jsonforms.umd.cjs"],
+            staticDir=os.path.join(os.path.dirname(__file__), "web_client", "dist"),
+            tree=info["serverRoot"],
+        )

girder_jsonforms/lib/google_drive.py

@@ -0,0 +1,157 @@
+import os
+import pickle
+import logging
+
+from google.auth.transport.requests import Request
+
+# from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaIoBaseUpload
+
+logger = logging.getLogger("google_drive")
+logprint = logger
+# If modifying these scopes, delete the file token.pickle.
+SCOPES = ["https://www.googleapis.com/auth/drive"]
+
+
+def get_credentials():
+    """Gets valid user credentials from storage.
+
+    If nothing has been stored, or if the stored credentials are invalid,
+    the OAuth2 flow is completed to obtain the new credentials.
+
+    Returns:
+        Credentials, the obtained credential.
+    """
+    creds = None
+    # The file token.pickle stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "token.pickle")
+
+    if os.path.exists(path):
+        with open(path, "rb") as token:
+            creds = pickle.load(token)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            raise ValueError("No valid credentials found")
+            # flow = InstalledAppFlow.from_client_secrets_file("credentials.json", SCOPES)
+            # creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open(path, "wb") as token:
+            pickle.dump(creds, token)
+    return creds
+
+
+def authenticate_gdrive():
+    # Load credentials from a file or any other secure source
+    # Replace 'path_to_credentials.json' with the actual path to your credentials file
+    credentials = get_credentials()
+    return build("drive", "v3", credentials=credentials)
+
+
+def create_folders(service, folder_id, path):
+    if service is None:
+        logger.exception("Service not initialized. Failed to create folder.")
+        return
+    current_folder_id = folder_id
+    folders = path.split("/")
+    for folder in folders:
+        if folder:
+            # Check if folder already exists
+            query = f"'{current_folder_id}' in parents and mimeType='application/vnd.google-apps.folder' and name='{folder}' and trashed=false"
+            response = (
+                service.files()
+                .list(
+                    q=query,
+                    fields="files(id)",
+                    includeItemsFromAllDrives=True,
+                    supportsAllDrives=True,
+                )
+                .execute()
+            )
+            existing_folders = response.get("files", [])
+            if existing_folders:
+                logprint.error(f"Folder '{folder}' already exists")
+                logprint.error(response)
+                current_folder_id = existing_folders[0]["id"]
+            else:
+                logprint.error(f"Creating folder '{folder}'")
+                file_metadata = {
+                    "name": folder,
+                    "mimeType": "application/vnd.google-apps.folder",
+                    "parents": [current_folder_id],
+                }
+                folder_result = (
+                    service.files()
+                    .create(body=file_metadata, fields="id", supportsAllDrives=True)
+                    .execute()
+                )
+                logprint.error(
+                    f"Folder '{folder}' created with ID: {folder_result['id']}"
+                )
+                current_folder_id = folder_result["id"]
+
+    return current_folder_id
+
+
+def upload_file_to_gdrive(service, folder_id, path, file_handle, mimetype="text/plain"):
+    if service is None:
+        logger.exception("Service not initialized. Failed to upload file.")
+        return
+    file_path, file_name = os.path.split(path)
+    folder_id = create_folders(service, folder_id, file_path)
+    logprint.error(f"Uploading file '{file_name}' to folder '{folder_id}'")
+
+    # Check if file already exists in the folder
+    query = f"'{folder_id}' in parents and name='{file_name}' and trashed=false"
+    response = (
+        service.files()
+        .list(
+            q=query,
+            fields="files(id)",
+            includeItemsFromAllDrives=True,
+            supportsAllDrives=True,
+        )
+        .execute()
+    )
+    existing_files = response.get("files", [])
+    media = MediaIoBaseUpload(file_handle, mimetype=mimetype)
+    if existing_files:
+        file_id = existing_files[0]["id"]
+        file_result = (
+            service.files()
+            .update(fileId=file_id, media_body=media, supportsAllDrives=True)
+            .execute()
+        )
+        logprint.error(f'File {file_name} updated with ID: {file_result["id"]}')
+    else:
+        file_metadata = {"name": file_name, "parents": [folder_id]}
+        file_result = (
+            service.files()
+            .create(
+                body=file_metadata,
+                media_body=media,
+                fields="id",
+                supportsAllDrives=True,
+            )
+            .execute()
+        )
+        logprint.error(
+            f'File {file_metadata["name"]} uploaded with ID: {file_result["id"]}'
+        )
+    return file_result["id"]
+
+
+# Example usage
+if __name__ == "__main__":
+    # folder_id = "1WgY3HJ0bxzQc-IA_dBdiwb9Hvq6ho3HS"  # Replace with the actual folder ID
+    folder_id = "13u4Citrr09FETrPDn2gJmBHWeaO0ylji"
+    path = "subfolder/subfolder2/subfolder4/file.txt"
+    file_handle = open("file.txt", "rb")  # Replace 'file.txt' with the actual file path
+    service = authenticate_gdrive()
+    file_id = upload_file_to_gdrive(service, folder_id, path, file_handle)

girder_jsonforms/lib/jq.py

@@ -0,0 +1,134 @@
+from typing import Any
+
+# find all occurences of a key in a nested json
+def find_key_paths(json_data: dict, key: str, path: str = "") -> list[str]:
+    results = []
+    # Check if the input is a dictionary
+    if isinstance(json_data, dict):
+        # If the key is found, return the path to it
+        if key in json_data:
+            results.append(path + key)
+        # Recursively search through each value in the dictionary
+        for k, v in json_data.items():
+            results.extend(find_key_paths(v, key, path + k + "."))
+    # Check if the input is a list
+    elif isinstance(json_data, list):
+        # Recursively search through each item in the list
+        for i, item in enumerate(json_data):
+            if isinstance(item, dict):
+                results.extend(find_key_paths(item, key, path + f"[{i}]."))
+            elif isinstance(item, list):
+                for j, sub_item in enumerate(item):
+                    results.extend(find_key_paths(sub_item, key, path + f"[{i}][{j}]."))
+    # Return None if the key is not found
+    return results
+
+
+# method to change the value of the key using key notation .[]
+def get_value(json_data: dict, key: str) -> dict:
+    for path in key.split("."):
+        if path.startswith("["):
+            index = int(path[1:-1])
+            json_data = json_data[index]
+        else:
+            json_data = json_data[path]
+    return json_data
+
+
+def set_value(json_data: dict, key: str, value: Any) -> None:
+    for path in key.split(".")[:-1]:
+        if path.startswith("["):
+            index = int(path[1:-1])
+            json_data = json_data[index]
+        else:
+            json_data = json_data[path]
+    path = key.split(".")[-1]
+    json_data[path] = value
+
+
+def convert_to_jq_notation(data: dict, parent_key: str = "", sep: str = ".") -> dict:
+    """
+    Convert a nested dictionary or list into a flat dictionary with jq-style keys.
+
+    Args:
+        data (dict or list): The nested dictionary or list to flatten.
+        parent_key (str): The base key for recursion (used internally).
+        sep (str): The separator to use for keys (default is ".").
+
+    Returns:
+        dict: A flattened dictionary with jq-like notation keys.
+    """
+    items = []
+
+    if isinstance(data, dict):
+        for key, value in data.items():
+            new_key = f"{parent_key}{sep}{key}" if parent_key else key
+            items.extend(convert_to_jq_notation(value, new_key, sep).items())
+    elif isinstance(data, list):
+        for index, value in enumerate(data):
+            new_key = f"{parent_key}{sep}[{index}]"
+            items.extend(convert_to_jq_notation(value, new_key, sep).items())
+    else:
+        items.append((parent_key, data))
+
+    return dict(items)
+
+
+def parse_jq_notation(input_dict: dict) -> dict:
+    """
+    Convert a dictionary with jq-style keys into a nested dictionary.
+
+    Args:
+        input_dict (dict): The dictionary with jq-style keys.
+
+    Returns:
+        dict: A nested dictionary.
+    """
+
+    def is_array_index(key):
+        return key.startswith("[") and key.endswith("]") and key[1:-1].isdigit()
+
+    output = {}
+
+    for key, value in input_dict.items():
+        keys = key.split(".")
+        current = output
+
+        for i, part in enumerate(keys[:-1]):
+            if is_array_index(part):
+                # Convert string digits to integers for lists
+                part = int(part.strip("[]"))
+
+            if "nestedlist" in key:
+                print(current, key, part)
+            if isinstance(part, int):
+                # If current is not a list, convert it into one
+                if not isinstance(current, list):
+                    current[keys[i - 1]] = []
+                    current = current[keys[i - 1]]
+                # Extend the list to ensure the index exists
+
+                if i + 1 <= len(keys[:-1]) and is_array_index(keys[i+1]):
+                    append_object = []
+                else:
+                    append_object = {}
+
+                while len(current) <= part:
+                    # It has to be a placeholder for dictionary or list
+                    current.append(append_object)
+                current = current[part]
+            else:
+                if part not in current:
+                    # Create a list if the next key is a digit
+                    next_part = keys[i + 1]
+                    current[part] = [] if is_array_index(next_part) else {}
+                current = current[part]
+
+        # Handle the last key
+        last_key = keys[-1]
+        if is_array_index(last_key):
+            current.append(value)
+        else:
+            current[last_key] = value
+
+    return output

girder_jsonforms/models/entry.py

@@ -0,0 +1,245 @@
+import datetime
+import io
+import json
+import os
+
+from girder import events
+from girder.constants import AccessType
+from girder.models.folder import Folder
+from girder.models.item import Item
+from girder.models.model_base import Model
+from girder.models.upload import Upload
+from girder.utility import acl_mixin, JsonEncoder, RequestBodyStream
+
+
+class FormEntry(acl_mixin.AccessControlMixin, Model):
+    def initialize(self):
+        global GDRIVE_SERVICE
+        self.name = "entry"
+        # TODO: create indices for all pairs?
+        # self.ensureIndices(["formId", "data.sampleId"])
+        self.resourceColl = ("form", "jsonforms")
+        self.resourceParent = "formId"
+
+        self.exposeFields(
+            level=AccessType.READ,
+            fields=(
+                "_id",
+                "formId",
+                "folderId",
+                "data",
+                "created",
+                "updated",
+                "files",
+                "folders",
+            ),
+        )
+
+    def validate(self, doc):
+        return doc
+
+    def _getExtraPath(self, template, data):
+        # Define a safe set of built-in functions and variables
+        safe_globals = {"__builtins__": None}
+        safe_locals = {"data": data, "ord": ord}
+
+        # Evaluate the template
+        try:
+            result = eval(f'f"{template}"', safe_globals, safe_locals)
+            return result
+        except Exception as e:
+            print("Error:", e)
+            return None
+
+    def create(self, form, data, source, destination, creator):
+        now = datetime.datetime.utcnow()
+        unique_field = form.get("uniqueField")
+        entry = {
+            "formId": form["_id"],
+            "data": data,
+            "created": now,
+            "updated": now,
+            "folderId": destination["_id"],
+            "files": [],
+            "folders": [],
+        }
+
+        if existing := self.findOne(
+            {
+                "formId": form["_id"],
+                f"data.{unique_field}": data[unique_field],
+            }
+        ):
+            # Update the existing entry
+            entry.update(
+                {
+                    "_id": existing["_id"],
+                    "created": existing["created"],
+                    "files": existing["files"],
+                    "folders": existing["folders"],
+                }
+            )
+
+        # Move from temp to destination
+        path = entry["data"].get("targetPath")
+        known_targets = {
+            None: (
+                self.get_destination_folder(path, destination, creator),
+                entry["data"].get(unique_field),
+            )
+        }
+        if source is not None:
+            for child in Folder().childFolders(source, "folder", user=creator):
+                path = child.get("meta", {}).get("targetPath")
+                try:
+                    target, _ = known_targets[path]
+                except KeyError:
+                    target = self.get_destination_folder(path, destination, creator)
+                    known_targets[path] = target, child.get("meta", {}).get(unique_field)
+                child = self.unique(child, target)
+                Folder().move(child, target, "folder")
+                # TODO upload to GDrive
+                entry["folders"].append(child["_id"])
+
+            for child in Folder().childItems(source):
+                path = child.get("meta", {}).get("targetPath")
+                try:
+                    target, _ = known_targets[path]
+                except KeyError:
+                    target = self.get_destination_folder(path, destination, creator)
+                    known_targets[path] = target, child.get("meta", {}).get(unique_field)
+                child = self.unique(child, target)
+                child = Item().move(child, target)
+                for file in Item().childFiles(child):
+                    # Upload to GDrive
+                    gdrive_folder_id = child.get("meta", {}).get("gdriveFolderId")
+                    if gdrive_folder_id:
+                        events.daemon.trigger(
+                            "gdrive.upload",
+                            {
+                                "file": file,
+                                "gdriveFolderId": gdrive_folder_id,
+                                "path": os.path.join(path, file["name"]),
+                                "currentUser": creator,
+                            },
+                        )
+                entry["files"].append(child["_id"])
+            Folder().remove(source)
+
+        if not form.get("serialize", False):
+            return self.save(entry)
+
+        # Dump the entry into json file, by creating bytes buffer from json dump and
+        # Upload().uploadFromFile will create a file in each destination folder
+        if len(known_targets) > 1:
+            known_targets.pop(None)
+
+        processed = set()
+        for path, (target, uniqueId) in known_targets.items():
+            if target["_id"] in processed:
+                continue
+            path = path or entry["data"].get("targetPath")
+            with io.BytesIO(
+                json.dumps(
+                    entry, sort_keys=True, allow_nan=False, cls=JsonEncoder
+                ).encode("utf-8")
+            ) as f:
+                reference = {
+                    f"{unique_field}": uniqueId,
+                    "targetPath": path,
+                    "gdriveFolderId": form.get("gdriveFolderId"),
+                }
+                size = f.getbuffer().nbytes
+                upload = self._get_upload_for_entry(
+                    form["entryFileName"], target, creator, size, reference
+                )
+                # not really chunking here as JSON is small
+                upload = Upload().handleChunk(upload, RequestBodyStream(f, size))
+                if form.get("gdriveFolderId"):
+                    events.daemon.trigger(
+                        "gdrive.upload",
+                        {
+                            "file": upload,
+                            "gdriveFolderId": form["gdriveFolderId"],
+                            "gdriveFileId": reference.get("gdriveFileId"),
+                            "path": os.path.join(path, upload["name"]),
+                            "currentUser": creator,
+                        },
+                    )
+            processed.add(target["_id"])
+
+        return self.save(entry)
+
+    @staticmethod
+    def _get_upload_for_entry(fname, target, creator, size, reference):
+        if existing_item := Item().findOne({"name": fname, "folderId": target["_id"]}):
+            file = Item().childFiles(existing_item)[0]
+            if "gdriveFileId" in existing_item.get("meta", {}):
+                reference["gdriveFileId"] = existing_item["meta"]["gdriveFileId"]
+            reference["itemId"] = existing_item["_id"]
+            serialized_reference = json.dumps(
+                reference, sort_keys=True, allow_nan=False, cls=JsonEncoder
+            )
+            upload = Upload().createUploadToFile(
+                file=file, user=creator, size=size, reference=serialized_reference
+            )
+        else:
+            serialized_reference = json.dumps(
+                reference, sort_keys=True, allow_nan=False, cls=JsonEncoder
+            )
+            upload = Upload().createUpload(
+                user=creator,
+                name=fname,
+                parentType="folder",
+                parent=target,
+                size=size,
+                mimeType="application/json",
+                reference=serialized_reference,
+            )
+        return upload
+
+    @staticmethod
+    def get_destination_folder(path, root, user):
+        if path is None:
+            return root
+
+        destination = root
+        for subfolder in path.split(os.path.sep):
+            destination = Folder().createFolder(
+                destination,
+                subfolder,
+                parentType="folder",
+                creator=user,
+                reuseExisting=True,
+            )
+
+        return destination
+
+    @staticmethod
+    def unique(child, destination):
+        name = child["name"]
+        n = 0
+        checkName = True
+        while checkName:
+            q = {
+                "name": name,
+                "folderId": destination["_id"],
+                "_id": {"$ne": child["_id"]},
+            }
+            dupItem = Item().findOne(q, fields=["_id"])
+            q = {
+                "name": name,
+                "parentId": destination["_id"],
+                "parentCollection": "folder",
+            }
+            dupFolder = Folder().findOne(q, fields=["_id"])
+
+            if dupItem is None and dupFolder is None:
+                child["name"] = name
+                checkName = False
+            else:
+                n += 1
+                name = f"{child['name']} ({n})"
+
+        child["lowerName"] = child["name"].lower()
+        return child