pysodafair 0.1.62__py3-none-any.whl

pysoda/schema/subjects.json

@@ -0,0 +1,131 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "array",
+  "minItems": 1,
+  "maxItems": 28,
+  "items": [
+    {
+      "type": "object",
+      "properties": {
+        "subject_id": {
+          "type": "string",
+          "description": "Unique identifier for the subject. This should be a unique identifier for the subject within the dataset.",
+          "minLength": 1
+        },
+        "pool_id": {
+          "type": "string",
+          "description": "Unique identifier for the pool associated with the subject. This should be a unique identifier for the pool within the dataset."
+        },
+        "subject_experimental_group": {
+          "type": "string",
+          "description": "Experimental group associated with the subject. This should provide information about the experimental conditions or treatment groups to which the subject belongs."
+        },
+        "age": {
+          "type": "string",
+          "description": "Age of the subject. This should be a numeric value representing the age of the subject at the time of data collection."
+        },
+        "sex": {
+          "type": "string",
+          "description": "Sex of the subject."
+        },
+        "species": {
+          "type": "string",
+          "description": "Species of the subject. This should be the scientific name of the species."
+        },
+        "strain": {
+          "type": "string",
+          "description": "Strain of the subject. This should provide information about the genetic background of the subject."
+        },
+        "rrid_for_strain": {
+          "type": "string",
+          "description": "RRID for strain. This should be a unique identifier for the strain in the Research Resource Identifier (RRID) database."
+        },
+        "age_category": {
+          "type": "string",
+          "description": "Age category of the subject. This should provide information about the age range of the subject."
+        },
+        "also_in_dataset": {
+          "type": "string",
+          "description": "Indicates if the subject is also part of another dataset."
+        },
+        "member_of": {
+          "type": "string",
+          "description": "Indicates the group or pool the subject is a member of."
+        },
+        "metadata_only": {
+          "type": "string",
+          "description": "Indicates if this entry is metadata only."
+        },
+        "laboratory_internal_id": {
+          "type": "string",
+          "description": "Internal identifier used by the laboratory for the subject."
+        },
+        "date_of_birth": {
+          "type": "string",
+          "description": "Date of birth of the subject."
+        },
+        "age_range_min": {
+          "type": "string",
+          "description": "Minimum age range of the subject."
+        },
+        "age_range_max": {
+          "type": "string",
+          "description": "Maximum age range of the subject."
+        },
+        "body_mass": {
+          "type": "string",
+          "description": "Body mass of the subject."
+        },
+        "genotype": {
+          "type": "string",
+          "description": "Genotype of the subject."
+        },
+        "phenotype": {
+          "type": "string",
+          "description": "Phenotype of the subject."
+        },
+        "handedness": {
+          "type": "string",
+          "description": "Handedness of the subject."
+        },
+        "reference_atlas": {
+          "type": "string",
+          "description": "Reference atlas used for the subject."
+        },
+        "experimental_log_file_path": {
+          "type": "string",
+          "description": "Path to the experimental log file for the subject."
+        },
+        "experiment_date": {
+          "type": "string",
+          "description": "Date of the experiment involving the subject."
+        },
+        "disease_or_disorder": {
+          "type": "string",
+          "description": "Disease or disorder associated with the subject."
+        },
+        "intervention": {
+          "type": "string",
+          "description": "Intervention applied to the subject."
+        },
+        "disease_model": {
+          "type": "string",
+          "description": "Disease model associated with the subject."
+        },
+        "protocol_title": {
+          "type": "string",
+          "description": "Title of the protocol used for the subject."
+        },
+        "protocol_url_or_doi": {
+          "type": "string",
+          "description": "URL or DOI of the protocol used for the subject."
+        }
+      },
+      "required": ["subject_id"],
+      "additionalProperties": {
+        "type": "string",
+        "description": "Additional properties for the subject. This can include any other relevant information about the subject that is not covered by the predefined fields."
+      }
+    }
+  ]
+}

pysoda/schema/submission_schema.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "consortium_data_standard": {
+      "type": "string",
+      "description": "The consortium data standard used for the submssion. Examples include SDS, BIDS, etc."
+    },
+    "funding_consortium": {
+      "type": "string",
+      "description": "The name of the funding consortium that supported the research. Examples include BRAIN Initiative, Allen Institute, etc."
+    },
+    "award_number": {
+      "type": "string",
+      "description": "The award number associated with the funding. This is typically a unique identifier assigned by the funding agency."
+    },
+    "milestone_achieved": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "A list of milestones achieved in the research. This should include specific goals or objectives that were met during the project."
+    },
+    "milestone_completion_date": {
+      "type": "string",
+      "description": "The date when the milestone was completed. This should be in ISO 8601 format."
+    }
+  },
+  "required": []
+}

pysoda/utils/__init__.py

@@ -0,0 +1,9 @@
+from .upload_utils import generate_options_set, generating_locally, generating_on_ps, uploading_with_ps_account, uploading_to_existing_ps_dataset, can_resume_prior_upload, virtual_dataset_empty, get_dataset_id
+from .exceptions import PropertyNotSetError, ConfigProfileNotSet, FailedToFetchPennsieveDatasets, FailedToFetchPennsieveDatasets, PennsieveActionNoPermission, PennsieveDatasetCannotBeFound, EmptyDatasetError, LocalDatasetMissingSpecifiedFiles, validation_error_message, GenericUploadError, PennsieveUploadException, PennsieveDatasetNameTaken, PennsieveDatasetNameInvalid, PennsieveAccountInvalid, GenerateOptionsNotSet, PennsieveDatasetFilesInvalid, PennsieveAgentError, PennsieveAccountInformationFailedAuthentication
+from .pennsieveAgentUtils import connect_pennsieve_client
+from .schema_validation import validate_schema, get_sds_headers, get_schema_path
+from .config import format_agent_profile_name
+from .pennsieveUtils import get_dataset_id, check_forbidden_characters_ps, get_users_dataset_list
+from .authentication import get_access_token, create_request_headers
+from .metadata_utils import column_check, returnFileURL, remove_high_level_folder_from_path, double_extensions, get_name_extension
+from .time_utils import TZLOCAL

pysoda/utils/authentication.py

@@ -0,0 +1,381 @@
+import boto3
+import requests
+from os.path import expanduser, join, exists
+from configparser import ConfigParser
+from .config import format_agent_profile_name
+from configparser import ConfigParser
+import time
+import boto3
+from os import mkdir
+import time
+from .exceptions import ConfigProfileNotSet, PennsieveAccountInformationFailedAuthentication
+
+from ..constants import PENNSIEVE_URL
+
+from .logger import logger
+
+from .profile import create_unique_profile_name
+
+userpath = expanduser("~")
+configpath = join(userpath, ".pennsieve", "config.ini")
+
+# Variables for token caching
+
+cached_access_token = None
+last_fetch_time = 0
+TOKEN_CACHE_DURATION = 60 # Amount of time in seconds to cache the access token
+
+
+
+
+def get_access_token(api_key=None, api_secret=None):
+    """
+        Creates a temporary access token for utilizing the Pennsieve API. Reads the api token and secret from the Pennsieve config.ini file.
+        get cognito config . If no target profile name is provided the default profile is used. 
+    """
+    global cached_access_token, last_fetch_time, TOKEN_CACHE_DURATION # Variables used for token caching
+    global logger
+    current_time = time.time()
+
+    # If the cached_access_token is not None and the last fetch time is less than the cache duration, return the cached access token
+    if cached_access_token and current_time - last_fetch_time < TOKEN_CACHE_DURATION:
+        return cached_access_token
+    
+    r = requests.get(f"{PENNSIEVE_URL}/authentication/cognito-config")
+    r.raise_for_status()
+
+    cognito_app_client_id = r.json()["tokenPool"]["appClientId"]
+    cognito_region_name = r.json()["region"]
+
+    cognito_idp_client = boto3.client(
+        "cognito-idp",
+        region_name=cognito_region_name,
+        aws_access_key_id="",
+        aws_secret_access_key="",
+    )
+
+    # use the default profile values for auth if no api_key or api_secret is provided
+    if api_key is None or api_secret is None:
+        api_key = get_profile_name_from_api_key("api_token")
+        api_secret = get_profile_name_from_api_key("api_secret")
+
+
+
+
+    login_response = cognito_idp_client.initiate_auth(
+    AuthFlow="USER_PASSWORD_AUTH",
+    AuthParameters={"USERNAME": api_key, "PASSWORD": api_secret},
+    ClientId=cognito_app_client_id,
+    )
+
+    cached_access_token = login_response["AuthenticationResult"]["AccessToken"]
+    last_fetch_time = current_time
+
+    return cached_access_token
+        
+
+def clear_cached_access_token():
+    global cached_access_token, last_fetch_time
+    cached_access_token = None
+
+
+def get_cognito_userpool_access_token(email, password):
+    """
+    Creates a temporary access token for utilizing the Pennsieve API. Utilizes email and password to authenticate with the Pennsieve Cognito Userpool 
+    which provides higher privileges than the API token and secret flow.
+    """
+    
+
+    try:
+        response = requests.get(f"{PENNSIEVE_URL}/authentication/cognito-config")
+        response.raise_for_status()
+        cognito_app_client_id = response.json()["userPool"]["appClientId"]
+        cognito_region = response.json()["userPool"]["region"]
+        cognito_client = boto3.client(
+            "cognito-idp",
+            region_name=cognito_region,
+            aws_access_key_id="",
+            aws_secret_access_key="",
+        )
+    except Exception as e:
+        raise Exception(e) from e
+
+    try:
+        login_response = cognito_client.initiate_auth(
+            AuthFlow="USER_PASSWORD_AUTH",
+            AuthParameters={"USERNAME": email, "PASSWORD": password},
+            ClientId=cognito_app_client_id,
+        )
+    except Exception as e:
+        raise PennsieveAccountInformationFailedAuthentication("Invalid email or password") from e
+    try:
+        access_token = login_response["AuthenticationResult"]["AccessToken"]
+        response = requests.get(
+            f"{PENNSIEVE_URL}/user", headers={"Authorization": f"Bearer {access_token}"}
+        )
+        response.raise_for_status()
+    except Exception as e:
+        raise e
+
+    return access_token
+
+def get_profile_name_from_api_key(key):
+    config = ConfigParser()
+    config.read(configpath)
+    if "global" not in config:
+        raise Exception("Profile has not been set")
+
+    keyname = config["global"]["default_profile"]
+
+    if keyname in config and key in config[keyname]:
+        return config[keyname][key]
+    return None
+
+
+def bf_delete_account(keyname):
+    """
+    Args:
+        keyname: name of local Pennsieve account key (string)
+    Action:
+        Deletes account information from the Pennsieve config file
+    """
+    config = ConfigParser()
+    config.read(configpath)
+    config.remove_section(keyname)
+    with open(configpath, "w") as configfile:
+        config.write(configfile)
+
+    
+
+def bf_delete_default_profile():
+    config = ConfigParser()
+    config.read(configpath)
+
+    if "global" not in config:
+        return 
+    
+    config.remove_section("global")
+
+    with open(configpath, "w") as configfile:
+        config.write(configfile)
+
+
+
+
+def bf_add_account_username(keyname, key, secret):
+    """
+    Associated with 'Add account' button in 'Login to your Pennsieve account' section of SODA
+
+    Args:
+        keyname: Name of the account to be associated with the given credentials (string)
+        key: API key (string)
+        secret: API Secret (string)
+    Action:
+        Adds account to the Pennsieve configuration file (local machine)
+    """
+    global logger
+
+
+    # format the keyname to lowercase and replace '.' with '_'
+    formatted_account_name = format_agent_profile_name(keyname)
+    
+    bf_delete_default_profile()
+    try:
+        bfpath = join(userpath, ".pennsieve")
+        # Load existing or create new config file
+        config = ConfigParser()
+        if exists(configpath):
+            config.read(configpath)
+        elif not exists(bfpath):
+            mkdir(bfpath)
+
+        # Add agent section
+        agentkey = "agent"
+        if not config.has_section(agentkey):
+            config.add_section(agentkey)
+            config.set(agentkey, "port", "9000")
+            config.set(agentkey, "upload_workers", "10")
+            config.set(agentkey, "upload_chunk_size", "32")
+
+        # Add new account if it does not already exist
+        if not config.has_section(formatted_account_name):
+            config.add_section(formatted_account_name)
+
+        config.set(formatted_account_name, "api_token", key)
+        config.set(formatted_account_name, "api_secret", secret)
+
+        # set profile name in global section
+        if not config.has_section("global"):
+            config.add_section("global")
+            config.set("global", "default_profile", formatted_account_name)
+
+        
+        with open(configpath, "w") as configfile:
+            config.write(configfile)
+
+    except Exception as e:
+        raise e
+
+    # Check key and secret are valid, if not delete account from config
+    try:
+        get_access_token()
+    except Exception as e:
+        logger.error(e)
+        bf_delete_account(formatted_account_name)
+        raise PennsieveAccountInformationFailedAuthentication("Pkease check that the key name, key value, and secret value are entered properly.") from e
+
+    try:
+        if not config.has_section("global"):
+            config.add_section("global")
+
+        default_acc = config["global"]
+        default_acc["default_profile"] = keyname
+
+        with open(configpath, "w+") as configfile:
+            config.write(configfile)
+
+        return {"message": f"Successfully added account {formatted_account_name}"}
+
+    except Exception as e:
+        bf_delete_account(keyname)
+        raise e
+
+
+def delete_duplicate_keys(token, keyname):
+    try:
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {token}",
+        }
+
+        r = requests.get(f"{PENNSIEVE_URL}/token", headers=headers)
+        r.raise_for_status()
+
+        tokens = r.json()
+
+        for token in tokens:
+            if token["name"] == keyname:
+                r = requests.delete(f"{PENNSIEVE_URL}/token/{token['key']}", headers=headers)
+                r.raise_for_status()
+    except Exception as e:
+        raise e
+
+
+def create_pennsieve_api_key_secret(email, password, machine_username_specifier):
+
+    api_key = get_cognito_userpool_access_token(email, password)
+
+    # TODO: Send in computer and profile of computer from frontend to this endpoint and use it in this function
+    profile_name = create_unique_profile_name(api_key, machine_username_specifier)
+
+
+    delete_duplicate_keys(api_key, "SODA-Pennsieve")
+    delete_duplicate_keys(api_key, profile_name)
+
+
+    url = "https://api.pennsieve.io/token/"
+
+    payload = {"name": f"{profile_name}"}
+    headers = {
+        "Accept": "*/*",
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+    }
+
+    response = requests.request("POST", url, json=payload, headers=headers)
+    response.raise_for_status()
+    response = response.json()
+
+
+    # clear access token cache
+    clear_cached_access_token()
+
+    return { 
+        "success": "success", 
+        "key": response["key"], 
+        "secret": response["secret"], 
+        "name": profile_name
+    }
+
+
+def get_access_token(api_key=None, api_secret=None):
+    """
+        Creates a temporary access token for utilizing the Pennsieve API. Reads the api token and secret from the Pennsieve config.ini file.
+        get cognito config . If no target profile name is provided the default profile is used. 
+    """
+    global cached_access_token, last_fetch_time, TOKEN_CACHE_DURATION # Variables used for token caching
+    # global logger
+    current_time = time.time()
+
+    print("Current time:", current_time)
+
+
+
+    # If the cached_access_token is not None and the last fetch time is less than the cache duration, return the cached access token
+    if cached_access_token and current_time - last_fetch_time < TOKEN_CACHE_DURATION:
+        return cached_access_token
+    
+    print("Cached token not returned")
+    r = requests.get(f"{PENNSIEVE_URL}/authentication/cognito-config")
+    r.raise_for_status()
+    print("Cognito config response:", r.json())
+
+    cognito_app_client_id = r.json()["tokenPool"]["appClientId"]
+    cognito_region_name = r.json()["region"]
+
+    cognito_idp_client = boto3.client(
+        "cognito-idp",
+        region_name=cognito_region_name,
+        aws_access_key_id="",
+        aws_secret_access_key="",
+    )
+
+    print("boto3 client done")
+
+    # use the default profile values for auth if no api_key or api_secret is provided
+    if api_key is None or api_secret is None:
+        api_key = get_profile_name_from_api_key("api_token")
+        api_secret = get_profile_name_from_api_key("api_secret")
+
+
+    login_response = cognito_idp_client.initiate_auth(
+    AuthFlow="USER_PASSWORD_AUTH",
+    AuthParameters={"USERNAME": api_key, "PASSWORD": api_secret},
+    ClientId=cognito_app_client_id,
+    )
+
+    cached_access_token = login_response["AuthenticationResult"]["AccessToken"]
+    last_fetch_time = current_time
+
+    return cached_access_token
+
+def get_profile_name_from_api_key(key):
+    config = ConfigParser()
+    config.read(configpath)
+    if "global" not in config:
+        raise ConfigProfileNotSet("global")
+
+    keyname = config["global"]["default_profile"]
+
+    if keyname in config and key in config[keyname]:
+        return config[keyname][key]
+    return None
+
+
+def create_request_headers(ps_or_token):
+    """
+    Creates necessary HTTP headers for making Pennsieve API requests.
+    Input: 
+        ps: Pennsieve object for a user that has been authenticated
+    """
+    if type(ps_or_token) == str:
+        return {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {ps_or_token}",
+        }
+    
+    return {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {ps_or_token.get_user().session_token}",
+    }

pysoda/utils/config.py

@@ -0,0 +1,68 @@
+"""
+Purpose: Functions for dealing with the pennsieve config file located at ~/.pennsieve/config.ini 
+"""
+from ..constants import PENNSIEVE_URL
+
+
+def add_api_host_to_config(configparser, target_section_name, configpath):
+    """
+    Args:
+        configparser: configparser object
+        target_section_name: the section name to add the api_host to. Should be an account section. 
+        configpath: the path to the config file ( ~/.pennsieve/config.ini )
+    Action:
+        Adds api_host to the section of the configparser object if it does not exist in the given section.
+        If given a section name that is 'agent' it will do nothing.
+    """
+
+    # do not add the api_host key to the agent section of the config
+    if target_section_name == 'agent':
+        return 
+
+    if not configparser.has_option(target_section_name, "api_host"):
+        configparser.set(target_section_name, "api_host", PENNSIEVE_URL)
+
+    with open(configpath, "w+") as configfile:
+        configparser.write(configfile)
+
+
+
+def format_agent_profile_name(profile_name):
+    """
+    Args:
+        profile_name: the profile name to format
+    Returns:
+        The formatted profile name
+    Notes:
+        We replace the '.' with '_' because the config parser on the Node side does not like '.' in the profile name.
+    """
+    return profile_name.lower().replace('.', '_').strip()
+
+
+def lowercase_account_names(config, account_name, configpath):
+    """
+    Args:
+        config: configparser object
+        account_name: the account name to convert to lowercase
+    Action:
+        Converts the account name and global default_profile value to lowercase and updates the config file.
+    """
+    formatted_account_name = format_agent_profile_name(account_name)
+    # if the section exists lowercased do nothing 
+    if config.has_section(formatted_account_name):
+        return
+
+    # add the section back with the lowercase account name
+    config.add_section(formatted_account_name) 
+    config.set(formatted_account_name, "api_token", config.get(account_name, "api_token"))
+    config.set(formatted_account_name, "api_secret", config.get(account_name, "api_secret"))
+
+    # set the global default_profile option to lowercase
+    config.set("global", "default_profile", formatted_account_name)
+
+    # remove the unformatted account name
+    config.remove_section(account_name)
+
+    # finalize the changes
+    with open(configpath, "w+") as configfile:
+        config.write(configfile)