caltechdata-api 1.10.6__tar.gz → 2.0.0__tar.gz

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caltechdata_api
-Version: 1.10.6
+Version: 2.0.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi
@@ -24,6 +24,7 @@ Requires-Dist: pyyaml
 Requires-Dist: s3fs
 Requires-Dist: cryptography
 Requires-Dist: s3cmd
+Requires-Dist: jsonschema-specifications
 Dynamic: license-file
 
 # CaltechDATA API Python Library

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api/caltechdata_edit.py

@@ -1,4 +1,4 @@
-import copy, os, json
+import copy, os, json, time
 
 import requests
 from requests import session
@@ -7,7 +7,6 @@ from caltechdata_api import (
     customize_schema,
     write_files_rdm,
     add_file_links,
-    send_to_community,
 )
 
 
@@ -16,7 +15,8 @@ def caltechdata_unembargo(token, ids, production=False):
 
 
 def caltechdata_accept(ids, token=None, production=False):
-    # Accept a record into a community
+    # Accept a record into a community. Only accepts the first community
+    # request
 
     # If no token is provided, get from RDMTOK environment variable
     if not token:
@@ -36,8 +36,15 @@ def caltechdata_accept(ids, token=None, production=False):
         result = requests.get(
             url + "/api/records/" + idv + "/draft/review", headers=headers
         )
-
-        accept_link = result.json()["links"]["actions"]["accept"]
+        if result.status_code != 200:
+            result = requests.get(
+                url + "/api/records/" + idv + "/requests", headers=headers
+            )
+            if result.status_code != 200:
+                raise Exception(result.text)
+            accept_link = result.json()["hits"]["hits"][0]["links"]["actions"]["accept"]
+        else:
+            accept_link = result.json()["links"]["actions"]["accept"]
         data = comment = {
             "payload": {
                 "content": "This record is accepted automatically with the CaltechDATA API",
@@ -97,7 +104,7 @@ def caltechdata_edit(
     token=None,
     files={},
     production=False,
-    schema="43",
+    schema=None,
     publish=False,
     file_links=[],
     s3=None,
@@ -122,41 +129,6 @@ def caltechdata_edit(
     if isinstance(files, str) == True:
         files = [files]
 
-    # Check if file links were provided in the metadata
-    descriptions = []
-    ex_file_links = []
-    ex_file_descriptions = []
-    if "descriptions" in metadata:
-        for d in metadata["descriptions"]:
-            if d["description"].startswith("Files available via S3"):
-                file_text = d["description"]
-                file_list = file_text.split('href="')
-                # Check if we have file_descriptions
-                split_comma = file_list[0].split(", ")
-                if len(split_comma) == 3:
-                    ex_file_descriptions.append(split_comma[1])
-                # Loop over links in description, skip header text
-                for file in file_list[1:]:
-                    ex_file_links.append(file.split('"\n')[0])
-                    split_comma = file.split(", ")
-                    if len(split_comma) == 3:
-                        ex_file_descriptions.append(split_comma[1])
-            else:
-                descriptions.append(d)
-        # We remove file link descriptions, and re-add below
-        metadata["descriptions"] = descriptions
-
-    # If user has provided file links as a cli option, we add those
-    if file_links:
-        metadata = add_file_links(
-            metadata, file_links, file_descriptions, s3_link=s3_link
-        )
-    # Otherwise we add file links found in the mtadata file
-    elif ex_file_links:
-        metadata = add_file_links(
-            metadata, ex_file_links, ex_file_descriptions, s3_link=s3_link
-        )
-
     if authors == False:
         if production == True:
             url = "https://data.caltech.edu/"
@@ -181,6 +153,18 @@ def caltechdata_edit(
         "Content-type": "application/octet-stream",
     }
 
+    # Add to community if provided:
+    if community:
+        result = requests.post(
+            url + "/api/records/" + idv + "/communities",
+            headers=headers,
+            data=json.dumps({"communities": [{"id": community}]}),
+        )
+        if result.status_code != 200:
+            print(result.url)
+            raise Exception(result.text)
+        caltechdata_accept([idv], token, production)
+
     # Check status
     existing = requests.get(
         url + "/api/records/" + idv,
@@ -193,7 +177,14 @@ def caltechdata_edit(
             headers=headers,
         )
         if existing.status_code != 200:
-            raise Exception(f"Record {idv} does not exist, cannot edit")
+            # Try back again to the record
+            existing = requests.get(
+                url + "/api/records/" + idv,
+                headers=headers,
+            )
+
+            if existing.status_code != 200:
+                raise Exception(f"Record {idv} does not exist, cannot edit")
 
     existing = existing.json()
     status = existing["status"]
@@ -287,10 +278,19 @@ def caltechdata_edit(
             # We want to have the system set new DOIs
             data["pids"] = {}
     else:
-        if authors == False:
+        if authors == False and schema == "43":
             metadata["pids"] = pids
             data = customize_schema.customize_schema(metadata, schema=schema)
-        else:
+        elif authors == False:
+            # Data using RDM schema, force oai PID
+            if "pids" not in metadata:
+                metadata["pids"] = {}
+            metadata["pids"]["oai"] = {
+                "identifier": f"oai:data.caltech.edu:{idv}",
+                "provider": "oai",
+            }
+            data = metadata
+        if authors == True:
             # Authors, force oai PID
             if "pids" not in metadata:
                 metadata["pids"] = {}
@@ -300,7 +300,21 @@ def caltechdata_edit(
             }
             data = metadata
 
-    if files:
+    # Check for existing draft
+    result = requests.get(
+        url + "/api/records/" + idv + "/draft",
+        headers=headers,
+    )
+    if result.status_code != 200:
+        # We make a draft
+        result = requests.post(
+            url + "/api/records/" + idv + "/draft",
+            headers=headers,
+        )
+        if result.status_code != 201:
+            raise Exception(result.text)
+
+    if files or file_links:
         if default_preview:
             data["files"] = {"enabled": True, "default_preview": default_preview}
         else:
@@ -313,30 +327,15 @@ def caltechdata_edit(
         )
         if result.status_code != 200:
             raise Exception(result.text)
-        file_link = result.json()["links"]["files"]
-        write_files_rdm(files, file_link, headers, f_headers, keepfiles=keepfiles)
+        file_upload_link = result.json()["links"]["files"]
+        if files:
+            write_files_rdm(
+                files, file_upload_link, headers, f_headers, keepfiles=keepfiles
+            )
+        if file_links:
+            add_file_links(file_upload_link, file_links, headers, keepfiles=keepfiles)
 
     else:
-        # Check for existing draft
-        result = requests.get(
-            url + "/api/records/" + idv + "/draft",
-            headers=headers,
-        )
-        if result.status_code != 200:
-            # We make a draft
-            result = requests.post(
-                url + "/api/records/" + idv + "/draft",
-                json=data,
-                headers=headers,
-            )
-            if result.status_code != 201:
-                raise Exception(result.text)
-            result = requests.get(
-                url + "/api/records/" + idv,
-                headers=headers,
-            )
-            if result.status_code != 200:
-                raise Exception(result.text)
         # We want files to stay the same as the existing record
         data["files"] = existing["files"]
         if default_preview:
@@ -348,13 +347,23 @@ def caltechdata_edit(
             json=data,
         )
         if result.status_code != 200:
-            raise Exception(result.text)
+            time.sleep(3)
+            result = requests.put(
+                url + "/api/records/" + idv + "/draft",
+                headers=headers,
+                json=data,
+            )
+            if result.status_code != 200:
+                raise Exception(result.text)
 
     if publish:
         publish_link = f"{url}/api/records/{idv}/draft/actions/publish"
         result = requests.post(publish_link, headers=headers)
         if result.status_code != 202:
-            raise Exception(result.text)
+            time.sleep(3)
+            result = requests.post(publish_link, headers=headers)
+            if result.status_code != 202:
+                raise Exception(result.text)
         if return_id:
             return result.json()["id"]
         else:

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api/caltechdata_write.py

@@ -67,45 +67,40 @@ def write_files_rdm(
                     raise Exception(result.text)
 
 
-def add_file_links(
-    metadata, file_links, file_descriptions=[], additional_descriptions="", s3_link=None
-):
+def add_file_links(file_upload_link, file_links, headers, keepfiles=False):
     # Currently configured for S3 links, assuming all are at the same endpoint
     link_string = ""
     endpoint = "https://" + file_links[0].split("/")[2]
     s3 = s3fs.S3FileSystem(anon=True, client_kwargs={"endpoint_url": endpoint})
     index = 0
+    file_json = []
+    f_list = []
     for link in file_links:
         file = link.split("/")[-1]
+        f_list.append(file)
         path = link.split(endpoint)[1]
         size = s3.info(path)["size"]
-        size = humanbytes(size)
-        try:
-            description = file_descriptions[index]
-            if description != " ":
-                desc = description + ","
-            else:
-                desc = ""
-        except IndexError:
-            desc = ""
-        if link_string == "":
-            if s3_link:
-                link_string = f"Files available via S3 at {s3_link}&lt;/p&gt;</p>"
-            else:
-                cleaned = link.strip(file)
-                link_string = f"Files available via S3 at {cleaned}&lt;/p&gt;</p>"
-        link_string += f"""{file}, {desc} {size}  
-        <p>&lt;a role="button" class="ui compact mini button" href="{link}"
-        &gt; &lt;i class="download icon"&gt;&lt;/i&gt; Download &lt;/a&gt;</p>&lt;/p&gt;</p>
-        """
-        index += 1
-    # Tack on any additional descriptions
-    if additional_descriptions != "":
-        link_string += additional_descriptions
 
-    description = {"description": link_string, "descriptionType": "files"}
-    metadata["descriptions"].append(description)
-    return metadata
+        file_json.append(
+            {"key": file, "size": size, "transfer": {"type": "R", "url": link}}
+        )
+    # Now we see if any existing files need to be replaced
+    result = requests.get(file_upload_link, headers=headers)
+    if result.status_code == 200:
+        ex_files = result.json()["entries"]
+        for ex in ex_files:
+            if ex["key"] in f_list:
+                result = requests.delete(ex["links"]["self"], headers=headers)
+                if result.status_code != 204:
+                    raise Exception(result.text)
+            else:
+                if keepfiles == False:
+                    result = requests.delete(ex["links"]["self"], headers=headers)
+                    if result.status_code != 204:
+                        raise Exception(result.text)
+    result = requests.post(file_upload_link, headers=headers, json=file_json)
+    if result.status_code != 201:
+        raise Exception(result.text)
 
 
 def send_to_community(
@@ -155,14 +150,13 @@ def caltechdata_write(
     token=None,
     files=[],
     production=False,
-    schema="43",
+    schema=None,
     publish=False,
     file_links=[],
     s3=None,
     community=None,
     authors=False,
     file_descriptions=[],
-    s3_link=None,
     default_preview=None,
     review_message=None,
     verify=True,
@@ -181,15 +175,25 @@ def caltechdata_write(
     if not token:
         token = os.environ["RDMTOK"]
 
+    if authors == False:
+        if production == True:
+            url = "https://data.caltech.edu/"
+        elif local == True:
+            url = "https://127.0.0.1:5000/"
+        else:
+            url = "https://data.caltechlibrary.dev/"
+    else:
+        if production == True:
+            url = "https://authors.library.caltech.edu/"
+        elif local == True:
+            url = "https://127.0.0.1:5000/"
+        else:
+            url = "https://authors.caltechlibrary.dev/"
+
     # If files is a string - change to single value array
     if isinstance(files, str) == True:
         files = [files]
 
-    if file_links:
-        metadata = add_file_links(
-            metadata, file_links, file_descriptions, s3_link=s3_link
-        )
-
     # Pull out pid information
     if production == True:
         repo_prefix = "10.22002"
@@ -235,22 +239,11 @@ def caltechdata_write(
     if "pids" not in metadata:
         metadata["pids"] = pids
 
-    if authors == False:
+    if authors == False and schema == "43":
         data = customize_schema.customize_schema(metadata, schema=schema)
-        if production == True:
-            url = "https://data.caltech.edu/"
-        elif local == True:
-            url = "https://127.0.0.1:5000/"
-        else:
-            url = "https://data.caltechlibrary.dev/"
     else:
+        # Using RDM schema, no customization
         data = metadata
-        if production == True:
-            url = "https://authors.library.caltech.edu/"
-        elif local == True:
-            url = "https://127.0.0.1:5000/"
-        else:
-            url = "https://authors.caltechlibrary.dev/"
 
     headers = {
         "Authorization": "Bearer %s" % token,
@@ -261,10 +254,13 @@ def caltechdata_write(
         "Content-type": "application/octet-stream",
     }
 
-    if not files:
+    if files or file_links:
+        if default_preview:
+            data["files"] = {"enabled": True, "default_preview": default_preview}
+        else:
+            data["files"] = {"enabled": True}
+    else:
         data["files"] = {"enabled": False}
-    elif default_preview:
-        data["files"] = {"enabled": True, "default_preview": default_preview}
 
     # Make draft and publish
     result = requests.post(
@@ -277,11 +273,14 @@ def caltechdata_write(
             raise Exception(result.text)
     idv = result.json()["id"]
     publish_link = result.json()["links"]["publish"]
+    file_link = result.json()["links"]["files"]
 
     if files:
-        file_link = result.json()["links"]["files"]
         write_files_rdm(files, file_link, headers, f_headers, s3, verify=verify)
 
+    if file_links:
+        add_file_links(file_link, file_links, headers)
+
     if community:
         review_link = result.json()["links"]["review"]
         send_to_community(

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api/cli.py

@@ -173,11 +173,9 @@ def get_funding_details():
     award_number = get_user_input("Enter the award number for funding: ")
     award_exists = check_award_number(award_number)
     if not award_exists:
-        print(
-            f"""Error: No award with number '{award_number}' found in
+        print(f"""Error: No award with number '{award_number}' found in
               CaltechDATA. You will need to provide more details about the
-              funding."""
-        )
+              funding.""")
     award_title = get_user_input("Enter the award title for funding: ")
     while True:
         funder_identifier = get_user_input("Enter the funder ROR (https://ror.org): ")
@@ -185,11 +183,9 @@ def get_funding_details():
         if name:
             break
         else:
-            print(
-                """This funder identifier is not a ROR. Please enter a valid
+            print("""This funder identifier is not a ROR. Please enter a valid
                   ROR identifier (without the url). For example the ROR for the
-                  NSF is 021nxhr62."""
-            )
+                  NSF is 021nxhr62.""")
     print("-" * 10)
     return {
         "awardNumber": award_number,
@@ -349,14 +345,12 @@ def write_s3cmd_config(endpoint):
         access_key = get_user_input("Enter the access key: ")
         secret_key = get_user_input("Enter the secret key: ")
         with open(configf, "w") as file:
-            file.write(
-                f"""[default]
+            file.write(f"""[default]
             access_key = {access_key}
             host_base = {endpoint}
             host_bucket = %(bucket).{endpoint}
             secret_key = {secret_key}
-            """
-            )
+            """)
 
 
 def upload_supporting_file(record_id=None):
@@ -370,8 +364,8 @@ def upload_supporting_file(record_id=None):
             "Do you want to upload or link data files? (upload/link/n): "
         ).lower()
         if choice == "link":
-            endpoint = "sdsc.osn.xsede.org"
-            path = "ini230004-bucket01/"
+            endpoint = "caltech2.osn.mghpcc.org"
+            path = "caltechdata-public/"
             if not record_id:
                 write_s3cmd_config(endpoint)
                 print("""S3 connection configured.""")
@@ -431,11 +425,9 @@ def upload_supporting_file(record_id=None):
                 elif filename in files:
                     file_size = os.path.getsize(filename)
                     if file_size > 1024 * 1024 * 1024:
-                        print(
-                            """The file is greater than 1 GB. Please upload the
+                        print("""The file is greater than 1 GB. Please upload the
                         metadata to CaltechDATA, and you'll be provided
-                        instructions to upload the files to S3 directly."""
-                        )
+                        instructions to upload the files to S3 directly.""")
                     else:
                         filepath = os.path.abspath(filename)
                         filepaths.append(filepath)
@@ -643,12 +635,10 @@ def print_upload_message(rec_id, production):
         if production
         else "https://data.caltechlibrary.dev/uploads/"
     )
-    print(
-        f"""You can view and publish this record at
+    print(f"""You can view and publish this record at
         {base_url}{rec_id}
         If you need to upload large files to S3, you can type
-        `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/`"""
-    )
+        `s3cmd put DATA_FILE s3://ini230004-bucket01/{rec_id}/`""")
 
 
 def edit_record(production):

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api/customize_schema.py

@@ -7,28 +7,6 @@ from pathlib import Path
 import requests
 
 
-def grid_to_ror(grid):
-    # Temporary until InvenioRDM stops spitting out GRIDS
-    # We manually handle some incorrect/redundant GRID Ids
-    if grid == "grid.451078.f":
-        ror = "00hm6j694"
-    elif grid == "grid.5805.8":
-        ror = "02en5vm52"
-    elif grid == "grid.465477.3":
-        ror = "00em52312"
-    else:
-        url = (
-            f"https://api.ror.org/organizations?query.advanced=external_ids.all:{grid}"
-        )
-        results = requests.get(url).json()
-        if len(results["items"]) == 0:
-            print(url + "doesn't have a valid ROR")
-            exit()
-        ror = results["items"][0]["id"]
-        ror = ror.split("ror.org/")[1]
-    return ror
-
-
 def get_vocabularies():
     """Returns dictionary of vocabularies"""
     path = Path(__file__).parent
@@ -297,7 +275,7 @@ def customize_schema_rdm(json_record):
             if "geoLocationPoint" in location:
                 lat = location["geoLocationPoint"]["pointLatitude"]
                 lon = location["geoLocationPoint"]["pointLongitude"]
-                new["geometry"] = {"type": "Point", "coordinates": [lat, lon]}
+                new["geometry"] = {"type": "Point", "coordinates": [lon, lat]}
             if "geoLocationBox" in location:
                 south = float(location["geoLocationBox"]["southBoundLatitude"])
                 north = float(location["geoLocationBox"]["northBoundLatitude"])
@@ -307,11 +285,11 @@ def customize_schema_rdm(json_record):
                     "type": "Polygon",
                     "coordinates": [
                         [
-                            [north, east],
-                            [north, west],
-                            [south, west],
-                            [south, east],
-                            [north, east],
+                            [east, north],
+                            [west, north],
+                            [west, south],
+                            [east, south],
+                            [east, north],
                         ]
                     ],
                 }
@@ -335,11 +313,6 @@ def customize_schema_rdm(json_record):
                         ror = ror.split("ror.org/")[1]
                     funder["id"] = ror
                     fund.pop("funderIdentifierType")
-                elif fund["funderIdentifierType"] == "GRID":
-                    # We need this temporarily to round-trip data
-                    ror = grid_to_ror(fund.pop("funderIdentifier"))
-                    funder["id"] = ror
-                    fund.pop("funderIdentifierType")
                 else:
                     print(f'Unknown Type mapping {fund["funderIdentifierType"]}')
             if "awardTitle" in fund:
@@ -568,8 +541,6 @@ def validate_metadata(json_record):
         else:
 
             for geo_loc in json_record["geoLocations"]:
-                if not isinstance(geo_loc, dict) or "geoLocationPlace" not in geo_loc:
-                    errors.append("Each 'geoLocation' must have 'geoLocationPlace'.")
                 if "geoLocationPoint" in geo_loc:
                     point = geo_loc["geoLocationPoint"]
                     if (

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api/get_metadata.py

@@ -12,11 +12,11 @@ def get_metadata(
     production=True,
     validate=True,
     emails=False,
-    schema="43",
+    schema=None,
     token=False,
     authors=False,
 ):
-    # Returns just DataCite metadata or DataCite metadata with emails
+    # Returns InvenioRDM metadata, or just DataCite metadata, or DataCite metadata with emails
 
     if production == True:
         if authors:
@@ -35,7 +35,7 @@ def get_metadata(
         "accept": "application/json",
     }
 
-    if authors:
+    if authors or schema == None:
         headers = base_headers
         validate = False
     else:
@@ -114,7 +114,7 @@ if __name__ == "__main__":
         action="store_true",
         help="skip validation of metadata",
     )
-    parser.add_argument("-schema", default="43", help="Schema Version")
+    parser.add_argument("-schema", default=None, help="Schema Version")
 
     args = parser.parse_args()
 

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caltechdata_api
-Version: 1.10.6
+Version: 2.0.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi
@@ -24,6 +24,7 @@ Requires-Dist: pyyaml
 Requires-Dist: s3fs
 Requires-Dist: cryptography
 Requires-Dist: s3cmd
+Requires-Dist: jsonschema-specifications
 Dynamic: license-file
 
 # CaltechDATA API Python Library

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/caltechdata_api.egg-info/requires.txt

@@ -5,3 +5,4 @@ pyyaml
 s3fs
 cryptography
 s3cmd
+jsonschema-specifications

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/setup.cfg

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [metadata]
 name = caltechdata_api
-version = 1.10.6
+version = 2.0.0
 author = Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi
 author_email = tmorrell@caltech.edu, aabakah@caltech.edu, knagi@caltech.edu
 description = Python wrapper for CaltechDATA API.
@@ -33,6 +33,7 @@ install_requires =
 	s3fs
 	cryptography
 	s3cmd
+	jsonschema-specifications
 include_package_data = True
 
 [options.packages.find]

{caltechdata_api-1.10.6 → caltechdata_api-2.0.0}/tests/test_rdm.py

@@ -68,8 +68,7 @@ def test_datacite_rdm_create_edit(full_datacite43_record):
 
     assert doi.startswith("10.33569")
 
-    # Validate is false until geolocation points are fixed/we move to 4.6
-    new_metadata = get_metadata(recid, production=False, validate=False)
+    new_metadata = get_metadata(recid, production=False, schema="43")
 
     assert new_metadata["publisher"] == "Edited"
 
@@ -89,7 +88,6 @@ def test_datacite_rdm_create_edit(full_datacite43_record):
 
     recid = new_doi.split("/")[1]
 
-    # Validate is false until geolocation points are fixed/we move to 4.6
-    new_metadata = get_metadata(recid, production=False, validate=False)
+    new_metadata = get_metadata(recid, production=False, schema="43")
 
     assert new_metadata["publisher"] == "Again!"