caltechdata-api 1.6.0__tar.gz → 1.6.1__tar.gz

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: caltechdata_api
-Version: 1.6.0
+Version: 1.6.1
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/caltechdata_api/cli.py

@@ -5,7 +5,7 @@ from caltechdata_api import caltechdata_write, caltechdata_edit
 from .md_to_json import parse_readme_to_json
 import json
 import os
-import configparser
+from cryptography.fernet import Fernet
 
 CALTECHDATA_API = "https://data.caltech.edu/api/names?q=identifiers.identifier:{}"
 ORCID_API = "https://orcid.org/"
@@ -21,26 +21,60 @@ funderIdentifier = ""
 funderIdentifierType = ""
 funderName = ""
 
+home_directory = os.path.expanduser("~")
+caltechdata_directory = os.path.join(home_directory, ".caltechdata")
 
-CONFIG_FILE = "caltechdata_config.ini"
 
+if not os.path.exists(caltechdata_directory):
+    os.makedirs(caltechdata_directory)
 
-def get_or_set_token():
-    config = configparser.ConfigParser()
 
-    if os.path.isfile(CONFIG_FILE):
-        config.read(CONFIG_FILE)
-        if "CaltechDATA" in config and "token" in config["CaltechDATA"]:
-            return config["CaltechDATA"]["token"]
+def generate_key():
+    return Fernet.generate_key()
+
+
+# Load the key from a file or generate a new one if not present
+def load_or_generate_key():
+    key_file = os.path.join(caltechdata_directory, "key.key")
+    if os.path.exists(key_file):
+        with open(key_file, "rb") as f:
+            return f.read()
     else:
+        key = generate_key()
+        with open(key_file, "wb") as f:
+            f.write(key)
+        return key
+
+
+# Encrypt the token
+def encrypt_token(token, key):
+    f = Fernet(key)
+    return f.encrypt(token.encode())
+
+
+# Decrypt the token
+def decrypt_token(encrypted_token, key):
+    f = Fernet(key)
+    return f.decrypt(encrypted_token).decode()
+
+
+# Function to get or set token
+def get_or_set_token():
+    key = load_or_generate_key()
+    token_file = os.path.join(caltechdata_directory, "token.txt")
+    try:
+        with open(token_file, "rb") as f:
+            encrypted_token = f.read()
+            token = decrypt_token(encrypted_token, key)
+            return token
+    except FileNotFoundError:
         while True:
-            token = get_user_input("Enter your CaltechDATA token: ")
-            confirm_token = get_user_input("Confirm your CaltechDATA token: ")
+            token = input("Enter your CaltechDATA token: ").strip()
+            confirm_token = input("Confirm your CaltechDATA token: ").strip()
             if token == confirm_token:
-                config.add_section("CaltechDATA")
-                config.set("CaltechDATA", "token", token)
-                with open(CONFIG_FILE, "w") as configfile:
-                    config.write(configfile)
+                encrypted_token = encrypt_token(token, key)
+                with open(token_file, "wb") as f:
+                    f.write(encrypted_token)
                 return token
             else:
                 print("Tokens do not match. Please try again.")
@@ -216,26 +250,42 @@ def get_names(orcid):
     return family_name, given_name
 
 
+def write_s3cmd_config(access_key, secret_key, endpoint):
+    configf = os.path.join(home_directory, ".s3cfg")
+    if not os.path.exists(key_file):
+        with open(configf, "w") as file:
+            file.write(
+                f"""[default]
+            access_key = {access_key}
+            host_base = {endpoint}
+            host_bucket = %(bucket).{endpoint}
+            secret_key = {secret_key}
+            """
+            )
+
+
 def upload_supporting_file(record_id=None):
     filepath = ""
-    file_link = ""
+    filepaths = []
+    file_links = []
     while True:
         choice = get_user_input(
             "Do you want to upload or link data files? (upload/link/n): "
         ).lower()
         if choice == "link":
-            endpoint = "https://sdsc.osn.xsede.org/"
+            endpoint = "sdsc.osn.xsede.org"
             path = "ini230004-bucket01/"
-
             if not record_id:
-                record_id = get_user_input("Folder where OSN files are uploaded")
-
+                access_key = get_user_input("Enter the access key: ")
+                secret_key = get_user_input("Enter the secret key: ")
+                write_s3cmd_config(access_key, secret_key, endpoint)
+                print("""S3 connection configured.""")
+                break
+            endpoint = f"https://{endpoint}/"
             s3 = s3fs.S3FileSystem(anon=True, client_kwargs={"endpoint_url": endpoint})
             # Find the files
             files = s3.glob(path + record_id + "/*")
 
-            file_links = []
-
             for link in files:
                 fname = link.split("/")[-1]
                 if "." not in fname:
@@ -264,33 +314,38 @@ def upload_supporting_file(record_id=None):
                 f for f in os.listdir() if not f.endswith(".json") and os.path.isfile(f)
             ]
             print("\n".join(files))
-            filename = get_user_input(
-                "Enter the filename to upload as a supporting file: "
-            )
-            if filename in files:
-                file_size = os.path.getsize(filename)
-                if file_size > 1024 * 1024 * 1024:
-                    file_link = get_user_input(
-                        "Enter the S3 link to the file (File size is more than 1GB): "
-                    )
-                    if file_link:
-                        return filepath, file_link
+            while True:
+                filename = get_user_input(
+                    "Enter the filename to upload as a supporting file (or 'n' to finish): "
+                )
+                if filename == "n":
+                    break
+                if filename in files:
+                    file_size = os.path.getsize(filename)
+                    if file_size > 1024 * 1024 * 1024:
+                        print(
+                            """The file is greater than 1 GB. Please upload the
+                        metadata to CaltechDATA, and you'll be provided
+                        instructions to upload the files to S3 directly."""
+                        )
                     else:
-                        print("Link is required for files larger than 1GB.")
-                        continue
+                        filepath = os.path.abspath(filename)
+                        filepaths.append(filepath)
                 else:
-                    filepath = os.path.abspath(filename)
-                    break
-            else:
-                print(
-                    f"Error: File '{filename}' not found. Please enter a valid filename."
-                )
+                    print(
+                        f"Error: File '{filename}' not found. Please enter a valid filename."
+                    )
+
+            add_more = get_user_input("Do you want to add more files? (y/n): ").lower()
+            if add_more != "y":
+                break
+
         elif choice == "n":
             break
         else:
             print("Invalid input. Please enter 'link' or 'upload' or 'n'.")
 
-    return filepath, file_link
+    return filepaths, file_links
 
 
 def upload_data_from_file():
@@ -361,7 +416,9 @@ def create_record():
                     )
                 rec_id = response
                 print(
-                    f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+                    f"""You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}
+                    If you need to upload large files to S3, you can type
+                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
                 )
                 break
             else:
@@ -424,7 +481,9 @@ def create_record():
                     )
                 rec_id = response
                 print(
-                    f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+                    f"""You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}
+                    If you need to upload large files to S3, you can type
+                    `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/"""
                 )
                 with open(response + ".json", "w") as file:
                     json.dump(metadata, file, indent=2)
@@ -474,7 +533,7 @@ def edit_record():
             )
         rec_id = response
         print(
-            f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}"
+            f"You can view and publish this record at https://data.caltechlibrary.dev/uploads/{rec_id}\n"
         )
 
 

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/caltechdata_api/md_to_json.py

@@ -33,6 +33,7 @@ def expand_special_keys(key, value):
             {
                 "nameIdentifier": orcid,
                 "nameIdentifierScheme": "ORCID",
+                "schemeUri": f"https://orcid.org/{value}",
             }
         ]
     return value
@@ -55,13 +56,20 @@ def parse_readme_to_json(readme_path):
     else:
         json_data["titles"] = [{"title": title_line.replace("# ", "")}]
 
+    contributors = []
+    identifiers = []
+    item_list = []
+
     section_pattern = re.compile(r"^##\s+(.*)$")
     key_value_pattern = re.compile(r"^-\s+(.*?):\s+(.*)$")
     link_pattern = re.compile(r"\[.*?\]\((.*?)\)")
 
     for line_number, line in enumerate(lines, 1):
         if not line.strip():
-            if current_object and current_section:
+            if item_list and current_section:
+                json_data[current_section] = item_list
+                item_list = []
+            elif current_object and current_section:
                 if current_section == "types":
                     json_data[current_section] = current_object
                 elif len(current_object) == 1:
@@ -70,6 +78,12 @@ def parse_readme_to_json(readme_path):
                         json_data[current_section] = value
                     else:
                         json_data[current_section].append(current_object)
+                elif current_section in ["creators", "contributors"]:
+                    contributors.append(current_object)
+                    current_object = {}
+                elif current_section == "identifiers":
+                    identifiers.append(current_object)
+                    current_object = {}
                 else:
                     json_data[current_section].append(current_object)
                 current_object = {}
@@ -77,7 +91,30 @@ def parse_readme_to_json(readme_path):
 
         section_match = section_pattern.match(line)
         if section_match:
-            if current_section and current_object:
+            if item_list:
+                json_data[current_section] = item_list
+            elif current_object:
+                if current_section in json_data:
+                    if isinstance(json_data[current_section], list):
+                        json_data[current_section].append(current_object)
+                    elif isinstance(json_data[current_section], dict):
+                        json_data[current_section].update(current_object)
+                else:
+                    json_data[current_section] = (
+                        [current_object]
+                        if current_section != "types"
+                        else current_object
+                    )
+                current_object = {}
+
+            elif contributors and current_section in ["creators", "contributors"]:
+                json_data[current_section] = contributors
+                contributors = []
+            elif identifiers and current_section == "identifiers":
+                json_data[current_section] = identifiers
+                identifiers = []
+
+            elif current_section and current_object:
                 if current_section == "types":
                     json_data[current_section] = current_object
                 elif len(current_object) == 1:
@@ -100,19 +137,38 @@ def parse_readme_to_json(readme_path):
 
             if key in ["affiliation", "nameIdentifiers"]:
                 value = expand_special_keys(key, value)
+            elif (
+                key == "nameType"
+                and current_object
+                and current_section in ["creators", "contributors"]
+            ):
+                contributors.append(current_object)
+                current_object = {}
+            elif current_section in ["subjects"]:
+                item_list.append({key: value})
+            elif current_section == "dates":
+                if key == "date":
+                    current_object["date"] = value
+                elif key == "dateType":
+                    current_object["dateType"] = value
+                    item_list.append(current_object)
+                    current_object = {}
             else:
                 link_match = link_pattern.search(value)
                 if link_match:
                     value = link_match.group(1)
-
-            current_object[key] = value
+                current_object[key] = value
 
         elif line.strip() and not section_match:
             raise ReadmeFormatException(
                 f"Incorrect format detected at line {line_number}: {line}"
             )
 
-    if current_section and current_object:
+    if contributors and current_section in ["creators", "contributors"]:
+        json_data[current_section] = contributors
+    elif identifiers and current_section == "identifiers":
+        json_data[current_section] = identifiers
+    elif current_section and current_object:
         if current_section == "types":
             json_data[current_section] = current_object
         elif len(current_object) == 1:
@@ -128,7 +184,7 @@ def parse_readme_to_json(readme_path):
 
 
 if __name__ == "__main__":
-    readme_path = "exampleREADME.md"
+    readme_path = "/Users/elizabethwon/downloads/exampleREADME.md"
     try:
         json_data = parse_readme_to_json(readme_path)
         output_json_path = "output1.json"

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/caltechdata_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: caltechdata-api
-Version: 1.6.0
+Version: 1.6.1
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/caltechdata_api.egg-info/requires.txt

@@ -3,5 +3,5 @@ datacite>1.1.0
 tqdm>=4.62.3
 pyyaml
 s3fs
-configparser
-awscli
+cryptography
+s3cmd

{caltechdata_api-1.6.0 → caltechdata_api-1.6.1}/setup.py

@@ -66,8 +66,8 @@ REQUIRED = [
     "tqdm>=4.62.3",
     "pyyaml",
     "s3fs",
-    "configparser",
-    "awscli",
+    "cryptography",
+    "s3cmd",
 ]
 
 # What packages are optional?