pysodafair 0.1.62__py3-none-any.whl

pysoda/utils/exceptions.py

@@ -0,0 +1,156 @@
+
+# create a custom exception that indicates a property in an object has not been set
+class PropertyNotSetError(Exception):
+    def __init__(self, property_name):
+        self.property_name = property_name
+        self.error_message = f"The property {self.property_name} has not been set."
+
+    def __str__(self):
+        return self.error_message
+    
+
+# create a custom exception that indicates that the 'PennsieveAgent' could not be started
+class PennsieveAgentError(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+    
+class FailedToFetchPennsieveDatasets(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+    
+class PennsieveDatasetCannotBeFound(Exception):
+    def __init__(self, dataset_name):
+        self.dataset_name = dataset_name
+        self.error_message = f"The Pennsieve dataset {self.dataset_name} could not be found."
+
+    def __str__(self):
+        return self.error_message
+    
+    
+class ConfigProfileNotSet(Exception):
+    def __init__(self, profile_name):
+        self.profile_name = profile_name
+        self.error_message = f"The profile {self.profile_name} has not been set."
+
+    def __str__(self):
+        return self.error_message
+    
+class GenerateOptionsNotSet(Exception):
+    def __init__(self):
+        self.error_message = "The generate options have not been set."
+
+    def __str__(self):
+        return self.error_message
+    
+
+class PennsieveActionNoPermission(Exception):
+    def __init__(self, action):
+        self.action = action
+        self.error_message = f"Do not have the correct permissions to perform action: {self.action} ."
+
+    def __str__(self):
+        return self.error_message
+
+class GenericUploadError(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+    
+
+class EmptyDatasetError(Exception):
+    def __init__(self, dataset_name, expanded=""):
+        self.dataset_name = dataset_name
+        self.expanded = expanded
+        self.error_message = f"The dataset {self.dataset_name} is empty. {expanded}"
+
+    def __str__(self):
+        return self.error_message
+    
+
+class LocalDatasetMissingSpecifiedFiles(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class PennsieveUploadException(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+        super().__init__(self.error_message)
+
+    def __str__(self):
+        return self.error_message
+
+
+class PennsieveAccountInformationFailedAuthentication(Exception):
+    def __init__(self, error_message):
+
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class PennsieveDatasetNameTaken(Exception):
+    def __init__(self, dataset_name):
+        self.dataset_name = dataset_name
+        self.error_message = f"The Pennsieve dataset name {self.dataset_name} is already taken."
+
+    def __str__(self):
+        return self.error_message
+    
+class PennsieveDatasetNameInvalid(Exception):
+    def __init__(self, dataset_name):
+        self.dataset_name = dataset_name
+        self.error_message = f"The Pennsieve dataset name {self.dataset_name} is invalid."
+
+    def __str__(self):
+        return self.error_message
+    
+class PennsieveAccountInvalid(Exception):
+    def __init__(self, account_name):
+        self.account_name = account_name
+        self.error_message = f"The Pennsieve account name {self.account_name} is invalid."
+
+    def __str__(self):
+        return self.error_message
+    
+class PennsieveDatasetFilesInvalid(Exception):
+    def __init__(self, error_message):
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+def validation_error_message(e):
+    """
+    Print a message for a validation error.
+    input: e (ValidationError): The validation error from the validate library.
+    output: human readable message for the validation error.
+    """
+    msg = "There following error was found in your metadata:"
+    e_type = e.schema_path.pop().strip()
+    print(e.schema_path)
+    if e_type == "type":
+        s = ''
+        while e.schema_path:
+            p_v = e.schema_path.popleft()
+            if p_v.strip() != "properties":
+                if s != '':
+                    s += ' -> '
+                s += p_v
+        msg = f"{msg} {s} needs to be a list of values."
+    if e_type == "required":
+        # peel out the first line from the stringified error message
+        msg = f"{msg} {e.message.splitlines()[0]}"
+    return msg

pysoda/utils/logger.py

@@ -0,0 +1,6 @@
+# filepath: c:\Users\aaron\pysoda\pysoda\utils\logger.py
+import logging
+
+# Create a logger for the package
+logger = logging.getLogger("pysoda.utils")
+logger.setLevel(logging.WARNING)  # Set the default log level

pysoda/utils/metadata_utils.py

@@ -0,0 +1,74 @@
+import os.path
+import requests
+from .authentication import create_request_headers
+from ..constants import PENNSIEVE_URL
+
+
+# check for non-empty fields (cells)
+def column_check(x):
+    return "unnamed" not in x.lower()
+
+
+# obtain Pennsieve S3 URL for an existing metadata file
+def returnFileURL(ps, item_id):
+    r = requests.get(f"{PENNSIEVE_URL}/packages/{item_id}/view", headers=create_request_headers(ps))
+    r.raise_for_status()
+
+    file_details = r.json()
+    file_id = file_details[0]["content"]["id"]
+    r = requests.get(
+        f"{PENNSIEVE_URL}/packages/{item_id}/files/{file_id}", headers=create_request_headers(ps)
+    )
+    r.raise_for_status()
+
+    file_url_info = r.json()
+    return file_url_info["url"]
+
+
+def remove_high_level_folder_from_path(paths):
+    """
+        Remove the high level folder from the path. This is necessary because the high level folder is not included in the manifest file name entry.
+    """
+
+    return "" if len(paths) == 1 else "/".join(paths[1:]) + "/"
+
+
+
+
+double_extensions = [
+    ".ome.tiff",
+    ".ome.tif",
+    ".ome.tf2,",
+    ".ome.tf8",
+    ".ome.btf",
+    ".ome.xml",
+    ".brukertiff.gz",
+    ".mefd.gz",
+    ".moberg.gz",
+    ".nii.gz",
+    ".mgh.gz",
+    ".tar.gz",
+    ".bcl.gz",
+]
+
+
+def get_name_extension(file_name):
+    double_ext = False
+    for ext in double_extensions:
+        if file_name.find(ext) != -1:
+            double_ext = True
+            break
+
+    ext = ""
+    name = ""
+
+    if double_ext == False:
+        name = os.path.splitext(file_name)[0]
+        ext = os.path.splitext(file_name)[1]
+    else:
+        ext = (
+            os.path.splitext(os.path.splitext(file_name)[0])[1]
+            + os.path.splitext(file_name)[1]
+        )
+        name = os.path.splitext(os.path.splitext(file_name)[0])[0]
+    return name, ext

pysoda/utils/pennsieveAgentUtils.py

@@ -0,0 +1,11 @@
+from pennsieve2.pennsieve import Pennsieve
+from .exceptions import PennsieveAgentError
+
+def connect_pennsieve_client(account_name):
+    """
+        Connects to Pennsieve Python client to the Agent and returns the initialized Pennsieve object.
+    """
+    try:
+        return Pennsieve(profile_name=account_name)
+    except Exception as e:
+        raise PennsieveAgentError(f"Could not connect to the Pennsieve agent: {e}")

pysoda/utils/pennsieveUtils.py

@@ -0,0 +1,118 @@
+import requests
+from ..constants import PENNSIEVE_URL
+from .authentication import get_access_token
+import re 
+from .exceptions import PennsieveDatasetCannotBeFound, FailedToFetchPennsieveDatasets
+
+def get_dataset_id(dataset_name_or_id):
+    """
+    Returns the dataset ID for the given dataset name.
+    If the dataset ID was provided instead of the name, the ID will be returned. *Common for Guided Mode*
+    
+    Input:
+        dataset_name_or_id: Pennsieve dataset name or ID to get the ID for
+    """
+    # If the input is already a dataset ID, return it
+    if dataset_name_or_id.startswith("N:dataset:"):
+        return dataset_name_or_id
+    
+
+    try:
+        # Attempt to retrieve the user's dataset list from Pennsieve
+        dataset_list = get_users_dataset_list()
+    except Exception as e:
+        raise FailedToFetchPennsieveDatasets(str(e))
+    
+    # Iterate through the user's dataset list to find a matching dataset name
+    for dataset in dataset_list:
+        if dataset["content"]["name"] == dataset_name_or_id:
+            return dataset["content"]["id"]
+    
+    # If no matching dataset is found, abort with a 404 status and a specific error message
+    raise PennsieveDatasetCannotBeFound(dataset_name_or_id)
+  
+
+def get_users_dataset_list():
+    """
+        Returns a list of datasets the user has access to.
+        Input:
+            token: Pennsieve access token
+    """
+
+    # The number of datasets to retrieve per chunk
+    NUMBER_OF_DATASETS_PER_CHUNK = 200
+    # The total number of datasets the user has access to (set after the first request)
+    NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO = None
+
+    # The offset is the number of datasets to skip before retrieving the next chunk of datasets (starts at 0, then increases by the number of datasets per chunk)
+    current_offset = 0
+    # The list of datasets the user has access to (datasets are added to this list after each request and then returned)
+    datasets = []
+
+    try:
+        # Get the first chunk of datasets as well as the total number of datasets the user has access to
+        r = requests.get(f"{PENNSIEVE_URL}/datasets/paginated", headers=create_request_headers(get_access_token()), params={"offset": current_offset, "limit": NUMBER_OF_DATASETS_PER_CHUNK})
+        r.raise_for_status()
+        responseJSON = r.json()
+        datasets.extend(responseJSON["datasets"])
+        NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO = responseJSON["totalCount"]
+
+        # If the number of datasets the user has access to is less than the number of datasets per chunk, we don't need to retrieve any more datasets
+        if NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO < NUMBER_OF_DATASETS_PER_CHUNK:
+            return datasets
+        
+        # Otherwise, we need to retrieve the rest of the datasets.
+        # We do this by retrieving chunks of datasets until the number of datasets retrieved is equal to the number of datasets the user has access to
+        while len(datasets) < NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO:
+            # Increase the offset by the number of datasets per chunk (e.g. if 200 datasets per chunk, then increase the offset by 200)
+            current_offset += NUMBER_OF_DATASETS_PER_CHUNK
+            r = requests.get(f"{PENNSIEVE_URL}/datasets/paginated", headers=create_request_headers(get_access_token()), params={"offset": current_offset, "limit": NUMBER_OF_DATASETS_PER_CHUNK})
+            r.raise_for_status()
+            responseJSON = r.json()
+            datasets.extend(responseJSON["datasets"])
+
+        return datasets
+    except Exception as e:
+        raise e
+    
+
+
+
+
+
+def create_request_headers(ps_or_token):
+    """
+    Creates necessary HTTP headers for making Pennsieve API requests.
+    Input: 
+        ps: Pennsieve object for a user that has been authenticated
+    """
+    if type(ps_or_token) == str:
+        return {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {ps_or_token}",
+        }
+    
+    return {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {ps_or_token.get_user().session_token}",
+    }
+
+
+forbidden_characters_bf = '\/:*?"<>.,'
+
+
+def check_forbidden_characters_ps(my_string):
+    """
+    Check for forbidden characters in Pennsieve file/folder name
+
+    Args:
+        my_string: string with characters (string)
+    Returns:
+        False: no forbidden character
+        True: presence of forbidden character(s)
+    """
+    regex = re.compile(f"[{forbidden_characters_bf}]")
+    if regex.search(my_string) == None and "\\" not in r"%r" % my_string:
+        return False
+    else:
+        return True

pysoda/utils/profile.py

@@ -0,0 +1,28 @@
+import requests
+from .config import format_agent_profile_name
+from ..constants import PENNSIEVE_URL
+def create_unique_profile_name(token, machine_username_specifier):
+    try:
+        # get the users email
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {token}",
+        }
+
+
+        r = requests.get(f"{PENNSIEVE_URL}/user", headers=headers)    
+        r.raise_for_status()    
+
+        user_info = r.json()
+
+        # create a substring of the start of the email to the @ symbol
+        email = user_info["email"]
+        email_sub = email.split("@")[0]
+
+        organization_id = user_info["preferredOrganization"]
+
+        # create an updated profile name that is unqiue to the user and their workspace 
+        return format_agent_profile_name(f"soda-pennsieve-{machine_username_specifier}-{email_sub}-{organization_id.lower()}")
+    except Exception as e:
+        raise e

pysoda/utils/schema_validation.py

@@ -0,0 +1,133 @@
+import json
+from jsonschema import validate
+import sys 
+import os
+
+
+
+def load_schema(schema_name):
+    schema_path = get_schema_path(schema_name)
+    with open(schema_path, 'r') as schema_file:
+        schema = json.load(schema_file)
+    return schema
+
+
+def get_schema_path(filename):
+    """Get the path to a schema file within the metadata_templates package."""
+    
+    # Method 1: Try PyInstaller bundle first (onefolder creates _MEIPASS)
+    if hasattr(sys, '_MEIPASS'):
+        # PyInstaller onefolder extracts to _MEIPASS/
+        possible_paths = [
+            os.path.join(sys._MEIPASS, "pysoda", "schema", filename),
+            os.path.join(sys._MEIPASS, filename)
+        ]
+        for path in possible_paths:
+            if os.path.exists(path):
+                return path
+    
+    # Method 2: Try to import the metadata_templates module (works if PyPI package is properly installed)
+    try:
+        from .. import schema
+        schema_dir = os.path.dirname(schema.__file__)
+        schema_path = os.path.join(schema_dir, filename)
+        if os.path.exists(schema_path):
+            return schema_path
+    except (ImportError, ModuleNotFoundError, AttributeError):
+        pass
+    
+    # Method 3: Search in the Flask app's directory structure
+    current_file = os.path.abspath(__file__)
+    current_dir = os.path.dirname(current_file)
+    
+    search_paths = [
+        os.path.join(current_dir, '..', '..', 'schema', filename),
+        os.path.join(current_dir, 'schema', filename),
+    ]
+    
+    for path in search_paths:
+        if os.path.exists(path):
+            return path
+        
+    
+    # Method 4: Use importlib_resources (Python 3.7+)
+    try:
+        from importlib import resources
+        with resources.path('schema', filename) as schema_path:
+            if schema_path.exists():
+                return str(schema_path)
+    except (ImportError, ModuleNotFoundError):
+        # Fallback to other methods if importlib_resources is not available
+        pass
+
+
+
+        # Method 5: Try to find in Electron Resources folder
+    try:
+        # Find the Electron Resources folder
+        current_path = current_dir
+        resources_folder = None
+        
+        # Walk up the directory tree to find the Resources folder
+        while current_path and current_path != os.path.dirname(current_path):
+            # Check common Electron Resources locations
+            possible_resources = [
+                os.path.join(current_path, 'Resources'),  # macOS
+                os.path.join(current_path, 'resources'),  # Windows/Linux
+                os.path.join(current_path, 'Contents', 'Resources'),  # macOS app bundle
+            ]
+            
+            for resource_path in possible_resources:
+                if os.path.exists(resource_path):
+                    resources_folder = resource_path
+                    break
+            
+            if resources_folder:
+                break
+                
+            current_path = os.path.dirname(current_path)
+        
+        # If we found the Resources folder, look for schema inside it
+        if resources_folder:
+            template_path = os.path.join(resources_folder, 'schema', filename)
+
+            if os.path.exists(template_path):
+                return template_path
+                
+    except Exception as e:
+        pass
+    
+    raise ImportError(f"Could not locate or create schema file {filename}.")
+
+
+# TODO: Make an enum of the schema names and add extensions to the schema names in the function.....or to the enum.
+def validate_schema(schema, schema_name):
+    """
+    Validate submission metadata against the submission schema.
+
+    Args:
+        schema (dict): The python dictionary version of the schema or subschema to validate against the json schema.
+        schema_name (str): The file name of the schema to validate against.
+
+    Raises:
+        ValidationError: If the metadata is invalid.
+    """
+    true_schema = load_schema(schema_name)
+    validate(instance=schema, schema=true_schema)
+
+
+def get_sds_headers(schema_name):
+    """
+    Get the headers for the SDS file.
+
+    Args:
+        soda (dict): The soda object containing the metadata.
+        schema_name (str): The name of the schema to validate against.
+
+    Returns:
+        list: The headers for the SDS file.
+    """
+
+    true_schema = load_schema(schema_name)
+    sds_headers = true_schema["items"][0]["properties"].keys()
+    return sds_headers

pysoda/utils/time_utils.py

@@ -0,0 +1,5 @@
+import datetime 
+
+### Internal functions
+def TZLOCAL():
+    return datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo

pysoda/utils/upload_utils.py

@@ -0,0 +1,108 @@
+import requests
+from os.path import expanduser, join
+from .exceptions import FailedToFetchPennsieveDatasets, PennsieveDatasetCannotBeFound
+from .authentication import get_access_token, create_request_headers
+
+
+from ..constants import PENNSIEVE_URL
+
+userpath = expanduser("~")
+configpath = join(userpath, ".pennsieve", "config.ini")
+
+def generate_options_set(soda_json_structure):
+    return "generate-dataset" in soda_json_structure.keys()
+
+def generating_locally(soda_json_structure):
+    return soda_json_structure["generate-dataset"]["destination"] == "local"
+
+def generating_on_ps(soda_json_structure):
+    return soda_json_structure["generate-dataset"]["destination"] == "ps"
+
+def uploading_with_ps_account(soda_json_structure):
+    return "ps-account-selected" in soda_json_structure
+
+def uploading_to_existing_ps_dataset(soda_json_structure):
+    return "ps-dataset-selected" in soda_json_structure
+
+def can_resume_prior_upload(resume_status):
+    global ums 
+    return resume_status and ums.df_mid_has_progress()
+
+def virtual_dataset_empty(soda_json_structure):
+    return (
+        "dataset-structure" not in soda_json_structure
+        and "metadata-files" not in soda_json_structure
+        )
+
+def get_dataset_id(dataset_name_or_id):
+    """
+    Returns the dataset ID for the given dataset name.
+    If the dataset ID was provided instead of the name, the ID will be returned. *Common for Guided Mode*
+    
+    Input:
+        dataset_name_or_id: Pennsieve dataset name or ID to get the ID for
+    """
+    # If the input is already a dataset ID, return it
+    if dataset_name_or_id.startswith("N:dataset:"):
+        return dataset_name_or_id
+    
+    # Attempt to retrieve the user's dataset list from Pennsieve
+    dataset_list = get_users_dataset_list()    
+    
+    # Iterate through the user's dataset list to find a matching dataset name
+    for dataset in dataset_list:
+        if dataset["content"]["name"] == dataset_name_or_id:
+            return dataset["content"]["id"]
+    
+    # If no matching dataset is found, abort with a 404 status and a specific error message
+    raise PennsieveDatasetCannotBeFound(dataset_name_or_id)
+
+
+def get_users_dataset_list():
+    """
+        Returns a list of datasets the user has access to.
+        Input:
+            token: Pennsieve access token
+    """
+
+    # The number of datasets to retrieve per chunk
+    NUMBER_OF_DATASETS_PER_CHUNK = 200
+    # The total number of datasets the user has access to (set after the first request)
+    NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO = None
+
+    # The offset is the number of datasets to skip before retrieving the next chunk of datasets (starts at 0, then increases by the number of datasets per chunk)
+    current_offset = 0
+    # The list of datasets the user has access to (datasets are added to this list after each request and then returned)
+    datasets = []
+
+    try:
+        # Get the first chunk of datasets as well as the total number of datasets the user has access to
+        r = requests.get(f"{PENNSIEVE_URL}/datasets/paginated", headers=create_request_headers(get_access_token()), params={"offset": current_offset, "limit": NUMBER_OF_DATASETS_PER_CHUNK})
+        r.raise_for_status()
+        responseJSON = r.json()
+        datasets.extend(responseJSON["datasets"])
+        NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO = responseJSON["totalCount"]
+
+        # If the number of datasets the user has access to is less than the number of datasets per chunk, we don't need to retrieve any more datasets
+        if NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO < NUMBER_OF_DATASETS_PER_CHUNK:
+            return datasets
+        
+        # Otherwise, we need to retrieve the rest of the datasets.
+        # We do this by retrieving chunks of datasets until the number of datasets retrieved is equal to the number of datasets the user has access to
+        while len(datasets) < NUMBER_OF_DATASETS_USER_HAS_ACCESS_TO:
+            # Increase the offset by the number of datasets per chunk (e.g. if 200 datasets per chunk, then increase the offset by 200)
+            current_offset += NUMBER_OF_DATASETS_PER_CHUNK
+            r = requests.get(f"{PENNSIEVE_URL}/datasets/paginated", headers=create_request_headers(get_access_token()), params={"offset": current_offset, "limit": NUMBER_OF_DATASETS_PER_CHUNK})
+            r.raise_for_status()
+            responseJSON = r.json()
+            datasets.extend(responseJSON["datasets"])
+
+        return datasets
+    except Exception as e:
+        raise FailedToFetchPennsieveDatasets("Error: Failed to retrieve datasets from Pennsieve. Please try again later.")
+
+
+
+
+
+