caltechdata-api 1.7.1__tar.gz → 1.8.1__tar.gz

{caltechdata_api-1.7.1 → caltechdata_api-1.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: caltechdata_api
-Version: 1.7.1
+Version: 1.8.1
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Bhattarai Rohan , Won Elizabeth

{caltechdata_api-1.7.1 → caltechdata_api-1.8.1}/caltechdata_api/caltechdata_write.py

@@ -1,7 +1,7 @@
 import copy
 import json
-import os, requests
-
+import os
+import requests
 import s3fs
 from requests import session
 from json.decoder import JSONDecodeError
@@ -49,8 +49,6 @@ def write_files_rdm(files, file_link, headers, f_headers, s3=None, keepfiles=Fal
                 infile = open(name, "rb")
             else:
                 infile = open(f_list[name], "rb")
-            # size = infile.seek(0, 2)
-            # infile.seek(0, 0)  # reset at beginning
             result = requests.put(link, headers=f_headers, data=infile)
             if result.status_code != 200:
                 raise Exception(result.text)
@@ -68,7 +66,7 @@ def write_files_rdm(files, file_link, headers, f_headers, s3=None, keepfiles=Fal
 def add_file_links(
     metadata, file_links, file_descriptions=[], additional_descriptions="", s3_link=None
 ):
-    # Currently configured for S3 links, assuming all are at same endpoint
+    # Currently configured for S3 links, assuming all are at the same endpoint
     link_string = ""
     endpoint = "https://" + file_links[0].split("/")[2]
     s3 = s3fs.S3FileSystem(anon=True, client_kwargs={"endpoint_url": endpoint})

{caltechdata_api-1.7.1 → caltechdata_api-1.8.1}/caltechdata_api/cli.py

@@ -59,11 +59,14 @@ def decrypt_token(encrypted_token, key):
     return f.decrypt(encrypted_token).decode()
 
 
-# Function to get or set token
-def get_or_set_token():
-
+# Function to get or set token with support for test system
+def get_or_set_token(production=True):
     key = load_or_generate_key()
-    token_file = os.path.join(caltechdata_directory, "token.txt")
+
+    # Use different token files for production and test environments
+    token_filename = "token.txt" if production else "token_test.txt"
+    token_file = os.path.join(caltechdata_directory, token_filename)
+
     try:
         with open(token_file, "rb") as f:
             encrypted_token = f.read()
@@ -71,8 +74,12 @@ def get_or_set_token():
             return token
     except FileNotFoundError:
         while True:
-            token = input("Enter your CaltechDATA token: ").strip()
-            confirm_token = input("Confirm your CaltechDATA token: ").strip()
+            token = input(
+                f"Enter your {'Production' if production else 'Test'} CaltechDATA token: "
+            ).strip()
+            confirm_token = input(
+                f"Confirm your {'Production' if production else 'Test'} CaltechDATA token: "
+            ).strip()
             if token == confirm_token:
                 encrypted_token = encrypt_token(token, key)
                 with open(token_file, "wb") as f:
@@ -133,7 +140,7 @@ def get_funding_entries():
 def validate_funder_identifier(funder_identifier):
     response = requests.get(f"https://api.ror.org/organizations/{funder_identifier}")
     if response.status_code == 200:
-        return True
+        return response.json().get("name")
     else:
         return False
 
@@ -150,7 +157,8 @@ def get_funding_details():
     award_title = get_user_input("Enter the award title for funding: ")
     while True:
         funder_identifier = get_user_input("Enter the funder ROR (https://ror.org): ")
-        if validate_funder_identifier(funder_identifier):
+        name = validate_funder_identifier(funder_identifier)
+        if name:
             break
         else:
             print(
@@ -162,6 +170,7 @@ def get_funding_details():
     return {
         "awardNumber": award_number,
         "awardTitle": award_title,
+        "funderName": name,
         "funderIdentifier": funder_identifier,
         "funderIdentifierType": "ROR",
     }
@@ -187,9 +196,18 @@ def parse_arguments():
         if license_number.isdigit() and 1 <= int(license_number) <= 8:
             # Valid license number selected
             args["license"] = {
-                "1": "cc0-1.0",
-                "2": "cc-by-4.0",
-                "3": "cc-by-nc-4.0",
+                "1": {
+                    "rights": "Creative Commons Zero v1.0 Universal",
+                    "rightsIdentifier": "cc0-1.0",
+                },
+                "2": {
+                    "rights": "Creative Commons Attribution v4.0 Universal",
+                    "rightsIdentifier": "cc-by-4.0",
+                },
+                "3": {
+                    "rights": "Creative Commons Attribution Non-Commercial v4.0 Universal",
+                    "rightsIdentifier": "cc-by-nc-4.0",
+                },
             }[license_number]
             break
         else:
@@ -252,9 +270,11 @@ def get_names(orcid):
     return family_name, given_name
 
 
-def write_s3cmd_config(access_key, secret_key, endpoint):
+def write_s3cmd_config(endpoint):
     configf = os.path.join(home_directory, ".s3cfg")
     if not os.path.exists(configf):
+        access_key = get_user_input("Enter the access key: ")
+        secret_key = get_user_input("Enter the secret key: ")
         with open(configf, "w") as file:
             file.write(
                 f"""[default]
@@ -279,9 +299,7 @@ def upload_supporting_file(record_id=None):
             endpoint = "sdsc.osn.xsede.org"
             path = "ini230004-bucket01/"
             if not record_id:
-                access_key = get_user_input("Enter the access key: ")
-                secret_key = get_user_input("Enter the secret key: ")
-                write_s3cmd_config(access_key, secret_key, endpoint)
+                write_s3cmd_config(endpoint)
                 print("""S3 connection configured.""")
                 break
             endpoint = f"https://{endpoint}/"
@@ -376,20 +394,35 @@ def upload_data_from_file():
                 print(f"Error: Invalid JSON format in the file '{filename}'. {str(e)}")
 
 
+def parse_args():
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(description="CaltechDATA CLI tool.")
+    parser.add_argument(
+        "-test", action="store_true", help="Use test mode, sets production to False"
+    )
+    args = parser.parse_args()
+    return args
+
+
 def main():
+    args = parse_args()
+
+    production = not args.test  # Set production to False if -test flag is provided
+
     choice = get_user_input(
         "Do you want to create or edit a CaltechDATA record? (create/edit): "
     ).lower()
     if choice == "create":
-        create_record()
+        create_record(production)
     elif choice == "edit":
-        edit_record()
+        edit_record(production)
     else:
         print("Invalid choice. Please enter 'create' or 'edit'.")
 
 
-def create_record():
-    token = get_or_set_token()
+def create_record(production):
+    token = get_or_set_token(production)
+    # keep_file = input("Do you want to keep your existing files? (yes/no): ").lower() == "yes"
     print("Using CaltechDATA token:", token)
     while True:
         choice = get_user_input(
@@ -401,7 +434,11 @@ def create_record():
             if existing_data:
                 if filepath != "":
                     response = caltechdata_write(
-                        existing_data, token, filepath, production=True, publish=False
+                        existing_data,
+                        token,
+                        filepath,
+                        production=production,
+                        publish=False,
                     )
                 elif file_link != "":
                     response = caltechdata_write(
@@ -414,15 +451,10 @@ def create_record():
                     )
                 else:
                     response = caltechdata_write(
-                        existing_data, token, production=True, publish=False
+                        existing_data, token, production=production, publish=False
                     )
                 rec_id = response
-                print(
-                    f"""You can view and publish this record at
-                    https://data.caltech.edu/uploads/{rec_id}
-                    If you need to upload large files to S3, you can type
-                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
-                )
+                print_upload_message(rec_id, production)
                 break
             else:
                 print("Going back to the main menu.")
@@ -457,9 +489,7 @@ def create_record():
                 ],
                 "types": {"resourceType": "", "resourceTypeGeneral": "Dataset"},
                 "rightsList": [
-                    {
-                        "rightsIdentifier": args["license"],
-                    }
+                    args["license"],
                 ],
                 "fundingReferences": args["fundingReferences"],
                 "schemaVersion": "http://datacite.org/schema/kernel-4",
@@ -468,27 +498,23 @@ def create_record():
             if confirm_upload():
                 if filepath != "":
                     response = caltechdata_write(
-                        metadata, token, filepath, production=True, publish=False
+                        metadata, token, filepath, production=production, publish=False
                     )
                 elif file_link != "":
                     response = caltechdata_write(
                         metadata,
                         token,
                         file_links=[file_link],
-                        production=True,
+                        production=production,
                         publish=False,
                     )
                 else:
                     response = caltechdata_write(
-                        metadata, token, production=True, publish=False
+                        metadata, token, production=production, publish=False
                     )
                 rec_id = response
-                print(
-                    f"""You can view and publish this record at
-                    https://data.caltech.edu/uploads/{rec_id}
-                    If you need to upload large files to S3, you can type
-                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
-                )
+
+                print_upload_message(rec_id, production)
                 with open(response + ".json", "w") as file:
                     json.dump(metadata, file, indent=2)
                 break
@@ -498,17 +524,32 @@ def create_record():
             print("Invalid choice. Please enter 'existing' or 'create'.")
 
 
-def edit_record():
+def print_upload_message(rec_id, production):
+    base_url = (
+        "https://data.caltech.edu/uploads/"
+        if production
+        else "https://data.caltechlibrary.dev/uploads/"
+    )
+    print(
+        f"""You can view and publish this record at
+        {base_url}{rec_id}
+        If you need to upload large files to S3, you can type
+        `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/`"""
+    )
+
+
+def edit_record(production):
     record_id = input("Enter the CaltechDATA record ID: ")
-    token = get_or_set_token()
+    token = get_or_set_token(production)
     file_name = download_file_by_id(record_id, token)
+
     if file_name:
         try:
             # Read the edited metadata file
             with open(file_name, "r") as file:
                 metadata = json.load(file)
             response = caltechdata_edit(
-                record_id, metadata, token, production=True, publish=False
+                record_id, metadata, token, production=production, publish=False
             )
             if response:
                 print("Metadata edited successfully.")
@@ -520,31 +561,54 @@ def edit_record():
         print("No metadata file found.")
     choice = get_user_input("Do you want to add files? (y/n): ").lower()
     if choice == "y":
-        API_URL_TEMPLATE = "https://data.caltech.edu/api/records/{record_id}/files"
+        if production:
+            API_URL_TEMPLATE = "https://data.caltech.edu/api/records/{record_id}/files"
+            API_URL_TEMPLATE_DRAFT = (
+                "https://data.caltech.edu/api/records/{record_id}/draft/files"
+            )
+        else:
+            API_URL_TEMPLATE = (
+                "https://data.caltechlibrary.dev/api/records/{record_id}/files"
+            )
+            API_URL_TEMPLATE_DRAFT = (
+                "https://data.caltechlibrary.dev/api/records/{record_id}/draft/files"
+            )
+
         url = API_URL_TEMPLATE.format(record_id=record_id)
+        url_draft = API_URL_TEMPLATE_DRAFT.format(record_id=record_id)
 
-        API_URL_TEMPLATE2 = (
-            "https://data.caltech.edu/api/records/{record_id}/draft/files"
-        )
-        url2 = API_URL_TEMPLATE2.format(record_id=record_id)
-        response = requests.get(url)
-        response2 = requests.get(url2)
-        filepath, file_link = upload_supporting_file(record_id)
-        print(file_link)
-        if response.status_code == 404 and response2.status_code == 404:
+        headers = {
+            "accept": "application/json",
+        }
+
+        if token:
+            headers["Authorization"] = "Bearer %s" % token
+
+        response = requests.get(url, headers=headers)
+        response_draft = requests.get(url_draft, headers=headers)
+        data = response.json()
+        data_draft = response_draft.json()
+        # Check if 'entries' exists and its length
+        if (
+            len(data.get("entries", [])) == 0
+            and len(data_draft.get("entries", [])) == 0
+        ):
             keepfile = False
         else:
-            keepfile = input("Do you want to keep existing files? y/n: ")
-            if keepfile == "y":
-                keepfile = True
-            else:
-                keepfile = False
+            keepfile = (
+                input("Do you want to keep existing files? (y/n): ").lower() == "y"
+            )
+
+        filepath, file_link = upload_supporting_file(record_id)
+        if file_link:
+            print(file_link)
+
         if filepath != "":
             response = caltechdata_edit(
                 record_id,
                 token=token,
                 files=filepath,
-                production=True,
+                production=production,
                 publish=False,
                 keepfiles=keepfile,
             )
@@ -554,14 +618,13 @@ def edit_record():
                 metadata,
                 token=token,
                 file_links=file_link,
-                production=True,
+                production=production,
                 publish=False,
-                keepfile=keepfile,
+                keepfiles=keepfile,
             )
+
         rec_id = response
-        print(
-            f"You can view and publish this record at https://data.caltech.edu/uploads/{rec_id}\n"
-        )
+        print_upload_message(rec_id, production)
 
 
 def download_file_by_id(record_id, token=None):
@@ -576,7 +639,6 @@ def download_file_by_id(record_id, token=None):
 
     try:
         response = requests.get(url, headers=headers)
-
         if response.status_code != 200:
             # Might have a draft
             response = requests.get(
@@ -584,7 +646,21 @@ def download_file_by_id(record_id, token=None):
                 headers=headers,
             )
             if response.status_code != 200:
-                raise Exception(f"Record {record_id} does not exist, cannot edit")
+                url = f"https://data.caltechlibrary.dev/api/records/{record_id}"
+                response = requests.get(
+                    url,
+                    headers=headers,
+                )
+                if response.status_code != 200:
+                    # Might have a draft
+                    response = requests.get(
+                        url + "/draft",
+                        headers=headers,
+                    )
+                    if response.status_code != 200:
+                        raise Exception(
+                            f"Record {record_id} does not exist, cannot edit"
+                        )
         file_content = response.content
         file_name = f"downloaded_data_{record_id}.json"
         with open(file_name, "wb") as file:

{caltechdata_api-1.7.1 → caltechdata_api-1.8.1}/caltechdata_api/customize_schema.py

@@ -134,8 +134,9 @@ def rdm_creators_contributors(person_list, peopleroles):
 
 def customize_schema_rdm(json_record):
     # Get vocabularies used in InvenioRDM
-    vocabularies = get_vocabularies()
 
+    vocabularies = get_vocabularies()
+    validate_metadata(json_record)
     peopleroles = vocabularies["crr"]
     resourcetypes = vocabularies["rsrct"]
     descriptiontypes = vocabularies["dty"]
@@ -386,6 +387,152 @@ def customize_schema_rdm(json_record):
     return final
 
 
+def validate_metadata(json_record):
+    """
+    Validates the presence and structure of required fields in a CaltechDATA JSON record.
+    Raises an exception if any required field is missing or structured incorrectly.
+    """
+    errors = []
+
+    # Check for 'types' and 'resourceTypeGeneral'
+    if "types" not in json_record:
+        errors.append("'types' field is missing.")
+    elif not isinstance(json_record["types"], dict):
+        errors.append("'types' field should be a dictionary.")
+    elif "resourceTypeGeneral" not in json_record["types"]:
+        errors.append("'resourceTypeGeneral' field is missing in 'types'.")
+
+    # Check for 'title'
+    if "titles" not in json_record:
+        errors.append("'titles' field is missing.")
+    elif not isinstance(json_record["titles"], list) or len(json_record["titles"]) == 0:
+        errors.append("'titles' should be a non-empty list.")
+    else:
+        # Ensure each title is a dictionary with 'title' field
+        for title in json_record["titles"]:
+            if not isinstance(title, dict) or "title" not in title:
+                errors.append(
+                    "Each entry in 'titles' must be a dictionary with a 'title' key."
+                )
+
+    # Publication date is handled by customize function
+
+    # Check for 'creators'
+    if "creators" not in json_record:
+        errors.append("'creators' field is missing.")
+    elif (
+        not isinstance(json_record["creators"], list)
+        or len(json_record["creators"]) == 0
+    ):
+        errors.append("'creators' should be a non-empty list.")
+    else:
+        for creator in json_record["creators"]:
+            if not isinstance(creator, dict) or "name" not in creator:
+                errors.append(
+                    "Each creator in 'creators' must be a dictionary with a 'name' key."
+                )
+
+    # Check for 'contributors'
+    if "contributors" in json_record:
+        if not isinstance(json_record["contributors"], list):
+            errors.append("'contributors' should be a list.")
+        else:
+            for contributor in json_record["contributors"]:
+                if not isinstance(contributor, dict) or "name" not in contributor:
+                    errors.append(
+                        "Each contributor must be a dictionary with a 'name' key."
+                    )
+
+    # Check for 'resourceType'
+    if "resourceType" not in json_record["types"]:
+        errors.append("'resourceType' field is missing in 'types'.")
+    elif not isinstance(json_record["types"]["resourceType"], str):
+        errors.append("'resourceType' should be a string.")
+
+    # Check for 'identifiers'
+    if "identifiers" in json_record:
+        if not isinstance(json_record["identifiers"], list):
+            errors.append("'identifiers' should be a list.")
+        else:
+            for identifier in json_record["identifiers"]:
+                if (
+                    not isinstance(identifier, dict)
+                    or "identifier" not in identifier
+                    or "identifierType" not in identifier
+                ):
+                    errors.append(
+                        "Each identifier must be a dictionary with 'identifier' and 'identifierType' keys."
+                    )
+
+    # Check for 'subjects'
+    if "subjects" in json_record:
+        if not isinstance(json_record["subjects"], list):
+            errors.append("'subjects' should be a list.")
+        else:
+            for subject in json_record["subjects"]:
+                if not isinstance(subject, dict) or "subject" not in subject:
+                    errors.append(
+                        "Each subject must be a dictionary with a 'subject' key."
+                    )
+
+    # Check for 'relatedIdentifiers'
+    if "relatedIdentifiers" in json_record:
+        if not isinstance(json_record["relatedIdentifiers"], list):
+            errors.append("'relatedIdentifiers' should be a list.")
+        else:
+            for related_id in json_record["relatedIdentifiers"]:
+                if (
+                    not isinstance(related_id, dict)
+                    or "relatedIdentifier" not in related_id
+                ):
+                    errors.append(
+                        "Each relatedIdentifier must be a dictionary with a 'relatedIdentifier' key."
+                    )
+
+    # Check for 'rightsList'
+    if "rightsList" in json_record:
+        if not isinstance(json_record["rightsList"], list):
+            errors.append("'rightsList' should be a list.")
+        else:
+            for rights in json_record["rightsList"]:
+                if not isinstance(rights, dict) or "rights" not in rights:
+                    errors.append(
+                        "Each entry in 'rightsList' must be a dictionary with a 'rights' key."
+                    )
+
+    # Check for 'geoLocations'
+    if "geoLocations" in json_record:
+        if not isinstance(json_record["geoLocations"], list):
+            errors.append("'geoLocations' should be a list.")
+        else:
+            for location in json_record["geoLocations"]:
+                if not isinstance(location, dict):
+                    errors.append("Each entry in 'geoLocations' must be a dictionary.")
+                elif (
+                    "geoLocationPoint" not in location
+                    and "geoLocationBox" not in location
+                    and "geoLocationPlace" not in location
+                ):
+                    errors.append(
+                        "Each geoLocation entry must contain at least one of 'geoLocationPoint', 'geoLocationBox', or 'geoLocationPlace'."
+                    )
+
+    # Check for 'fundingReferences'
+    if "fundingReferences" in json_record:
+        if not isinstance(json_record["fundingReferences"], list):
+            errors.append("'fundingReferences' should be a list.")
+        else:
+            for funding in json_record["fundingReferences"]:
+                if not isinstance(funding, dict):
+                    errors.append("Each funding reference must be a dictionary.")
+                if "funderName" not in funding:
+                    errors.append("Each funding reference must contain 'funderName'.")
+
+    # Return errors if any are found
+    if errors:
+        raise ValueError(f"Validation errors in metadata: {', '.join(errors)}")
+
+
 if __name__ == "__main__":
     # Read in from file for demo purposes
 

{caltechdata_api-1.7.1 → caltechdata_api-1.8.1}/caltechdata_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: caltechdata-api
-Version: 1.7.1
+Version: 1.8.1
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Bhattarai Rohan , Won Elizabeth