caltechdata-api 1.6.0__tar.gz → 1.7.0__tar.gz

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: caltechdata_api
-Version: 1.6.0
+Version: 1.7.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
-Author: Thomas E Morrell
-Author-email: tmorrell@caltech.edu
+Author: Thomas E Morrell, Bhattarai Rohan , Won Elizabeth
+Author-email: tmorrell@caltech.edu, , 
 License: https://data.caltech.edu/license
 Platform: UNKNOWN
 Classifier: License :: OSI Approved :: BSD License

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api/caltechdata_edit.py

@@ -65,6 +65,7 @@ def caltechdata_edit(
     s3_link=None,
     default_preview=None,
     authors=False,
+    keepfiles=False,
 ):
     # Make a copy of the metadata to make sure our local changes don't leak
     metadata = copy.deepcopy(metadata)
@@ -246,7 +247,7 @@ def caltechdata_edit(
         if result.status_code != 200:
             raise Exception(result.text)
         file_link = result.json()["links"]["files"]
-        write_files_rdm(files, file_link, headers, f_headers)
+        write_files_rdm(files, file_link, headers, f_headers, keepfiles=keepfiles)
 
     else:
         # Check for existing draft

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api/caltechdata_write.py

@@ -9,7 +9,7 @@ from caltechdata_api import customize_schema
 from caltechdata_api.utils import humanbytes
 
 
-def write_files_rdm(files, file_link, headers, f_headers, s3=None):
+def write_files_rdm(files, file_link, headers, f_headers, s3=None, keepfiles=False):
     f_json = []
     f_list = {}
     fnames = []
@@ -59,9 +59,10 @@ def write_files_rdm(files, file_link, headers, f_headers, s3=None):
                 raise Exception(result.text)
         else:
             # Delete any files not included in this write command
-            result = requests.delete(self, headers=f_headers)
-            if result.status_code != 204:
-                raise Exception(result.text)
+            if keepfiles == False:
+                result = requests.delete(self, headers=f_headers)
+                if result.status_code != 204:
+                    raise Exception(result.text)
 
 
 def add_file_links(

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api/cli.py

@@ -5,7 +5,7 @@ from caltechdata_api import caltechdata_write, caltechdata_edit
 from .md_to_json import parse_readme_to_json
 import json
 import os
-import configparser
+from cryptography.fernet import Fernet
 
 CALTECHDATA_API = "https://data.caltech.edu/api/names?q=identifiers.identifier:{}"
 ORCID_API = "https://orcid.org/"
@@ -22,25 +22,61 @@ funderIdentifierType = ""
 funderName = ""
 
 
-CONFIG_FILE = "caltechdata_config.ini"
+home_directory = os.path.expanduser("~")
+caltechdata_directory = os.path.join(home_directory, ".caltechdata")
 
 
-def get_or_set_token():
-    config = configparser.ConfigParser()
+if not os.path.exists(caltechdata_directory):
+    os.makedirs(caltechdata_directory)
+
+
+def generate_key():
+    return Fernet.generate_key()
 
-    if os.path.isfile(CONFIG_FILE):
-        config.read(CONFIG_FILE)
-        if "CaltechDATA" in config and "token" in config["CaltechDATA"]:
-            return config["CaltechDATA"]["token"]
+
+# Load the key from a file or generate a new one if not present
+def load_or_generate_key():
+    key_file = os.path.join(caltechdata_directory, "key.key")
+    if os.path.exists(key_file):
+        with open(key_file, "rb") as f:
+            return f.read()
     else:
+        key = generate_key()
+        with open(key_file, "wb") as f:
+            f.write(key)
+        return key
+
+
+# Encrypt the token
+def encrypt_token(token, key):
+    f = Fernet(key)
+    return f.encrypt(token.encode())
+
+
+# Decrypt the token
+def decrypt_token(encrypted_token, key):
+    f = Fernet(key)
+    return f.decrypt(encrypted_token).decode()
+
+
+# Function to get or set token
+def get_or_set_token():
+
+    key = load_or_generate_key()
+    token_file = os.path.join(caltechdata_directory, "token.txt")
+    try:
+        with open(token_file, "rb") as f:
+            encrypted_token = f.read()
+            token = decrypt_token(encrypted_token, key)
+            return token
+    except FileNotFoundError:
         while True:
-            token = get_user_input("Enter your CaltechDATA token: ")
-            confirm_token = get_user_input("Confirm your CaltechDATA token: ")
+            token = input("Enter your CaltechDATA token: ").strip()
+            confirm_token = input("Confirm your CaltechDATA token: ").strip()
             if token == confirm_token:
-                config.add_section("CaltechDATA")
-                config.set("CaltechDATA", "token", token)
-                with open(CONFIG_FILE, "w") as configfile:
-                    config.write(configfile)
+                encrypted_token = encrypt_token(token, key)
+                with open(token_file, "wb") as f:
+                    f.write(encrypted_token)
                 return token
             else:
                 print("Tokens do not match. Please try again.")
@@ -216,26 +252,42 @@ def get_names(orcid):
     return family_name, given_name
 
 
+def write_s3cmd_config(access_key, secret_key, endpoint):
+    configf = os.path.join(home_directory, ".s3cfg")
+    if not os.path.exists(key_file):
+        with open(configf, "w") as file:
+            file.write(
+                f"""[default]
+            access_key = {access_key}
+            host_base = {endpoint}
+            host_bucket = %(bucket).{endpoint}
+            secret_key = {secret_key}
+            """
+            )
+
+
 def upload_supporting_file(record_id=None):
     filepath = ""
+    filepaths = []
     file_link = ""
+    file_links = []
     while True:
         choice = get_user_input(
             "Do you want to upload or link data files? (upload/link/n): "
         ).lower()
         if choice == "link":
-            endpoint = "https://sdsc.osn.xsede.org/"
+            endpoint = "sdsc.osn.xsede.org"
             path = "ini230004-bucket01/"
-
             if not record_id:
-                record_id = get_user_input("Folder where OSN files are uploaded")
-
+                access_key = get_user_input("Enter the access key: ")
+                secret_key = get_user_input("Enter the secret key: ")
+                write_s3cmd_config(access_key, secret_key, endpoint)
+                print("""S3 connection configured.""")
+                break
+            endpoint = f"https://{endpoint}/"
             s3 = s3fs.S3FileSystem(anon=True, client_kwargs={"endpoint_url": endpoint})
             # Find the files
             files = s3.glob(path + record_id + "/*")
-
-            file_links = []
-
             for link in files:
                 fname = link.split("/")[-1]
                 if "." not in fname:
@@ -264,33 +316,38 @@ def upload_supporting_file(record_id=None):
                 f for f in os.listdir() if not f.endswith(".json") and os.path.isfile(f)
             ]
             print("\n".join(files))
-            filename = get_user_input(
-                "Enter the filename to upload as a supporting file: "
-            )
-            if filename in files:
-                file_size = os.path.getsize(filename)
-                if file_size > 1024 * 1024 * 1024:
-                    file_link = get_user_input(
-                        "Enter the S3 link to the file (File size is more than 1GB): "
-                    )
-                    if file_link:
-                        return filepath, file_link
+            while True:
+                filename = get_user_input(
+                    "Enter the filename to upload as a supporting file (or 'n' to finish): "
+                )
+                if filename == "n":
+                    break
+                if filename in files:
+                    file_size = os.path.getsize(filename)
+                    if file_size > 1024 * 1024 * 1024:
+                        print(
+                            """The file is greater than 1 GB. Please upload the
+                        metadata to CaltechDATA, and you'll be provided
+                        instructions to upload the files to S3 directly."""
+                        )
                     else:
-                        print("Link is required for files larger than 1GB.")
-                        continue
+                        filepath = os.path.abspath(filename)
+                        filepaths.append(filepath)
                 else:
-                    filepath = os.path.abspath(filename)
+                    print(
+                        f"Error: File '{filename}' not found. Please enter a valid filename."
+                    )
+                add_more = get_user_input(
+                    "Do you want to add more files? (y/n): "
+                ).lower()
+                if add_more != "y":
                     break
-            else:
-                print(
-                    f"Error: File '{filename}' not found. Please enter a valid filename."
-                )
+            break
         elif choice == "n":
             break
         else:
             print("Invalid input. Please enter 'link' or 'upload' or 'n'.")
-
-    return filepath, file_link
+    return filepaths, file_links
 
 
 def upload_data_from_file():
@@ -344,7 +401,7 @@ def create_record():
             if existing_data:
                 if filepath != "":
                     response = caltechdata_write(
-                        existing_data, token, filepath, production=False, publish=False
+                        existing_data, token, filepath, production=True, publish=False
                     )
                 elif file_link != "":
                     response = caltechdata_write(
@@ -352,16 +409,19 @@ def create_record():
                         token,
                         file_links=[file_link],
                         s3_link=file_link,
-                        production=False,
+                        production=True,
                         publish=False,
                     )
                 else:
                     response = caltechdata_write(
-                        existing_data, token, production=False, publish=False
+                        existing_data, token, production=True, publish=False
                     )
                 rec_id = response
                 print(
-                    f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+                    f"""You can view and publish this record at
+                    https://data.caltech.edu/uploads/{rec_id}
+                    If you need to upload large files to S3, you can type
+                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
                 )
                 break
             else:
@@ -408,23 +468,26 @@ def create_record():
             if confirm_upload():
                 if filepath != "":
                     response = caltechdata_write(
-                        metadata, token, filepath, production=False, publish=False
+                        metadata, token, filepath, production=True, publish=False
                     )
                 elif file_link != "":
                     response = caltechdata_write(
                         metadata,
                         token,
                         file_links=[file_link],
-                        production=False,
+                        production=True,
                         publish=False,
                     )
                 else:
                     response = caltechdata_write(
-                        metadata, token, production=False, publish=False
+                        metadata, token, production=True, publish=False
                     )
                 rec_id = response
                 print(
-                    f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+                    f"""You can view and publish this record at
+                    https://data.caltech.edu/uploads/{rec_id}
+                    If you need to upload large files to S3, you can type
+                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
                 )
                 with open(response + ".json", "w") as file:
                     json.dump(metadata, file, indent=2)
@@ -445,7 +508,7 @@ def edit_record():
             with open(file_name, "r") as file:
                 metadata = json.load(file)
             response = caltechdata_edit(
-                record_id, metadata, token, production=False, publish=False
+                record_id, metadata, token, production=True, publish=False
             )
             if response:
                 print("Metadata edited successfully.")
@@ -457,11 +520,33 @@ def edit_record():
         print("No metadata file found.")
     choice = get_user_input("Do you want to add files? (y/n): ").lower()
     if choice == "y":
+        API_URL_TEMPLATE = "https://data.caltech.edu/api/records/{record_id}/files"
+        url = API_URL_TEMPLATE.format(record_id=record_id)
+
+        API_URL_TEMPLATE2 = (
+            "https://data.caltech.edu/api/records/{record_id}/draft/files"
+        )
+        url2 = API_URL_TEMPLATE2.format(record_id=record_id)
+        response = requests.get(url)
+        response2 = requests.get(url2)
         filepath, file_link = upload_supporting_file(record_id)
         print(file_link)
+        if response.status_code == 404 and response2.status_code == 404:
+            keepfile = False
+        else:
+            keepfile = input("Do you want to keep existing files? y/n: ")
+            if keepfile == "y":
+                keepfile = True
+            else:
+                keepfile = False
         if filepath != "":
             response = caltechdata_edit(
-                record_id, token=token, files=filepath, production=False, publish=False
+                record_id,
+                token=token,
+                files=filepath,
+                production=True,
+                publish=False,
+                keepfiles=keepfile,
             )
         elif file_link != "":
             response = caltechdata_edit(
@@ -469,17 +554,18 @@ def edit_record():
                 metadata,
                 token=token,
                 file_links=file_link,
-                production=False,
+                production=True,
                 publish=False,
+                keepfile=keepfile,
             )
         rec_id = response
         print(
-            f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+            f"You can view and publish this record at https://data.caltech.edu/uploads/{rec_id}\n"
         )
 
 
 def download_file_by_id(record_id, token=None):
-    url = f"https://data.caltechlibrary.dev/api/records/{record_id}"
+    url = f"https://data.caltech.edu/api/records/{record_id}"
 
     headers = {
         "accept": "application/vnd.datacite.datacite+json",
@@ -490,6 +576,7 @@ def download_file_by_id(record_id, token=None):
 
     try:
         response = requests.get(url, headers=headers)
+
         if response.status_code != 200:
             # Might have a draft
             response = requests.get(

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api/md_to_json.py

@@ -33,6 +33,7 @@ def expand_special_keys(key, value):
             {
                 "nameIdentifier": orcid,
                 "nameIdentifierScheme": "ORCID",
+                "schemeUri": f"https://orcid.org/{value}",
             }
         ]
     return value
@@ -55,13 +56,20 @@ def parse_readme_to_json(readme_path):
     else:
         json_data["titles"] = [{"title": title_line.replace("# ", "")}]
 
+    contributors = []
+    identifiers = []
+    item_list = []
+
     section_pattern = re.compile(r"^##\s+(.*)$")
     key_value_pattern = re.compile(r"^-\s+(.*?):\s+(.*)$")
     link_pattern = re.compile(r"\[.*?\]\((.*?)\)")
 
     for line_number, line in enumerate(lines, 1):
         if not line.strip():
-            if current_object and current_section:
+            if item_list and current_section:
+                json_data[current_section] = item_list
+                item_list = []
+            elif current_object and current_section:
                 if current_section == "types":
                     json_data[current_section] = current_object
                 elif len(current_object) == 1:
@@ -70,6 +78,12 @@ def parse_readme_to_json(readme_path):
                         json_data[current_section] = value
                     else:
                         json_data[current_section].append(current_object)
+                elif current_section in ["creators", "contributors"]:
+                    contributors.append(current_object)
+                    current_object = {}
+                elif current_section == "identifiers":
+                    identifiers.append(current_object)
+                    current_object = {}
                 else:
                     json_data[current_section].append(current_object)
                 current_object = {}
@@ -77,7 +91,30 @@ def parse_readme_to_json(readme_path):
 
         section_match = section_pattern.match(line)
         if section_match:
-            if current_section and current_object:
+            if item_list:
+                json_data[current_section] = item_list
+            elif current_object:
+                if current_section in json_data:
+                    if isinstance(json_data[current_section], list):
+                        json_data[current_section].append(current_object)
+                    elif isinstance(json_data[current_section], dict):
+                        json_data[current_section].update(current_object)
+                else:
+                    json_data[current_section] = (
+                        [current_object]
+                        if current_section != "types"
+                        else current_object
+                    )
+                current_object = {}
+
+            elif contributors and current_section in ["creators", "contributors"]:
+                json_data[current_section] = contributors
+                contributors = []
+            elif identifiers and current_section == "identifiers":
+                json_data[current_section] = identifiers
+                identifiers = []
+
+            elif current_section and current_object:
                 if current_section == "types":
                     json_data[current_section] = current_object
                 elif len(current_object) == 1:
@@ -100,6 +137,22 @@ def parse_readme_to_json(readme_path):
 
             if key in ["affiliation", "nameIdentifiers"]:
                 value = expand_special_keys(key, value)
+            elif (
+                key == "nameType"
+                and current_object
+                and current_section in ["creators", "contributors"]
+            ):
+                contributors.append(current_object)
+                current_object = {}
+            elif current_section in ["subjects"]:
+                item_list.append({key: value})
+            elif current_section == "dates":
+                if key == "date":
+                    current_object["date"] = value
+                elif key == "dateType":
+                    current_object["dateType"] = value
+                    item_list.append(current_object)
+                    current_object = {}
             else:
                 link_match = link_pattern.search(value)
                 if link_match:
@@ -112,7 +165,11 @@ def parse_readme_to_json(readme_path):
                 f"Incorrect format detected at line {line_number}: {line}"
             )
 
-    if current_section and current_object:
+    if contributors and current_section in ["creators", "contributors"]:
+        json_data[current_section] = contributors
+    elif identifiers and current_section == "identifiers":
+        json_data[current_section] = identifiers
+    elif current_section and current_object:
         if current_section == "types":
             json_data[current_section] = current_object
         elif len(current_object) == 1:
@@ -128,7 +185,7 @@ def parse_readme_to_json(readme_path):
 
 
 if __name__ == "__main__":
-    readme_path = "exampleREADME.md"
+    readme_path = "/Users/elizabethwon/downloads/exampleREADME.md"
     try:
         json_data = parse_readme_to_json(readme_path)
         output_json_path = "output1.json"

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: caltechdata-api
-Version: 1.6.0
+Version: 1.7.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
-Author: Thomas E Morrell
-Author-email: tmorrell@caltech.edu
+Author: Thomas E Morrell, Bhattarai Rohan , Won Elizabeth
+Author-email: tmorrell@caltech.edu, , 
 License: https://data.caltech.edu/license
 Platform: UNKNOWN
 Classifier: License :: OSI Approved :: BSD License

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/caltechdata_api.egg-info/requires.txt

@@ -3,5 +3,5 @@ datacite>1.1.0
 tqdm>=4.62.3
 pyyaml
 s3fs
-configparser
-awscli
+cryptography
+s3cmd

{caltechdata_api-1.6.0 → caltechdata_api-1.7.0}/setup.py

@@ -40,9 +40,9 @@ version = meta["version"]
 author = ""
 author_email = ""
 for obj in meta["author"]:
-    given = obj["givenName"]
-    family = obj["familyName"]
-    email = obj["email"]
+    given = obj.get("givenName", "")
+    family = obj.get("familyName", "")
+    email = obj.get("email", "")
     if len(author) == 0:
         author = given + " " + family
     else:
@@ -66,8 +66,8 @@ REQUIRED = [
     "tqdm>=4.62.3",
     "pyyaml",
     "s3fs",
-    "configparser",
-    "awscli",
+    "cryptography",
+    "s3cmd",
 ]
 
 # What packages are optional?