caltechdata-api 1.11.0__tar.gz → 2.0.0__tar.gz

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caltechdata_api
-Version: 1.11.0
+Version: 2.0.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api/caltechdata_edit.py

@@ -7,7 +7,6 @@ from caltechdata_api import (
     customize_schema,
     write_files_rdm,
     add_file_links,
-    send_to_community,
 )
 
 
@@ -16,7 +15,8 @@ def caltechdata_unembargo(token, ids, production=False):
 
 
 def caltechdata_accept(ids, token=None, production=False):
-    # Accept a record into a community
+    # Accept a record into a community. Only accepts the first community
+    # request
 
     # If no token is provided, get from RDMTOK environment variable
     if not token:
@@ -36,8 +36,15 @@ def caltechdata_accept(ids, token=None, production=False):
         result = requests.get(
             url + "/api/records/" + idv + "/draft/review", headers=headers
         )
-
-        accept_link = result.json()["links"]["actions"]["accept"]
+        if result.status_code != 200:
+            result = requests.get(
+                url + "/api/records/" + idv + "/requests", headers=headers
+            )
+            if result.status_code != 200:
+                raise Exception(result.text)
+            accept_link = result.json()["hits"]["hits"][0]["links"]["actions"]["accept"]
+        else:
+            accept_link = result.json()["links"]["actions"]["accept"]
         data = comment = {
             "payload": {
                 "content": "This record is accepted automatically with the CaltechDATA API",
@@ -97,7 +104,7 @@ def caltechdata_edit(
     token=None,
     files={},
     production=False,
-    schema="43",
+    schema=None,
     publish=False,
     file_links=[],
     s3=None,
@@ -122,41 +129,6 @@ def caltechdata_edit(
     if isinstance(files, str) == True:
         files = [files]
 
-    # Check if file links were provided in the metadata
-    descriptions = []
-    ex_file_links = []
-    ex_file_descriptions = []
-    if "descriptions" in metadata:
-        for d in metadata["descriptions"]:
-            if d["description"].startswith("Files available via S3"):
-                file_text = d["description"]
-                file_list = file_text.split('href="')
-                # Check if we have file_descriptions
-                split_comma = file_list[0].split(", ")
-                if len(split_comma) == 3:
-                    ex_file_descriptions.append(split_comma[1])
-                # Loop over links in description, skip header text
-                for file in file_list[1:]:
-                    ex_file_links.append(file.split('"\n')[0])
-                    split_comma = file.split(", ")
-                    if len(split_comma) == 3:
-                        ex_file_descriptions.append(split_comma[1])
-            else:
-                descriptions.append(d)
-        # We remove file link descriptions, and re-add below
-        metadata["descriptions"] = descriptions
-
-    # If user has provided file links as a cli option, we add those
-    if file_links:
-        metadata = add_file_links(
-            metadata, file_links, file_descriptions, s3_link=s3_link
-        )
-    # Otherwise we add file links found in the mtadata file
-    elif ex_file_links:
-        metadata = add_file_links(
-            metadata, ex_file_links, ex_file_descriptions, s3_link=s3_link
-        )
-
     if authors == False:
         if production == True:
             url = "https://data.caltech.edu/"
@@ -181,6 +153,18 @@ def caltechdata_edit(
         "Content-type": "application/octet-stream",
     }
 
+    # Add to community if provided:
+    if community:
+        result = requests.post(
+            url + "/api/records/" + idv + "/communities",
+            headers=headers,
+            data=json.dumps({"communities": [{"id": community}]}),
+        )
+        if result.status_code != 200:
+            print(result.url)
+            raise Exception(result.text)
+        caltechdata_accept([idv], token, production)
+
     # Check status
     existing = requests.get(
         url + "/api/records/" + idv,
@@ -294,10 +278,19 @@ def caltechdata_edit(
             # We want to have the system set new DOIs
             data["pids"] = {}
     else:
-        if authors == False:
+        if authors == False and schema == "43":
             metadata["pids"] = pids
             data = customize_schema.customize_schema(metadata, schema=schema)
-        else:
+        elif authors == False:
+            # Data using RDM schema, force oai PID
+            if "pids" not in metadata:
+                metadata["pids"] = {}
+            metadata["pids"]["oai"] = {
+                "identifier": f"oai:data.caltech.edu:{idv}",
+                "provider": "oai",
+            }
+            data = metadata
+        if authors == True:
             # Authors, force oai PID
             if "pids" not in metadata:
                 metadata["pids"] = {}
@@ -307,7 +300,21 @@ def caltechdata_edit(
             }
             data = metadata
 
-    if files:
+    # Check for existing draft
+    result = requests.get(
+        url + "/api/records/" + idv + "/draft",
+        headers=headers,
+    )
+    if result.status_code != 200:
+        # We make a draft
+        result = requests.post(
+            url + "/api/records/" + idv + "/draft",
+            headers=headers,
+        )
+        if result.status_code != 201:
+            raise Exception(result.text)
+
+    if files or file_links:
         if default_preview:
             data["files"] = {"enabled": True, "default_preview": default_preview}
         else:
@@ -320,29 +327,15 @@ def caltechdata_edit(
         )
         if result.status_code != 200:
             raise Exception(result.text)
-        file_link = result.json()["links"]["files"]
-        write_files_rdm(files, file_link, headers, f_headers, keepfiles=keepfiles)
+        file_upload_link = result.json()["links"]["files"]
+        if files:
+            write_files_rdm(
+                files, file_upload_link, headers, f_headers, keepfiles=keepfiles
+            )
+        if file_links:
+            add_file_links(file_upload_link, file_links, headers, keepfiles=keepfiles)
 
     else:
-        # Check for existing draft
-        result = requests.get(
-            url + "/api/records/" + idv + "/draft",
-            headers=headers,
-        )
-        if result.status_code != 200:
-            # We make a draft
-            result = requests.post(
-                url + "/api/records/" + idv + "/draft",
-                headers=headers,
-            )
-            if result.status_code != 201:
-                time.sleep(3)
-                result = requests.post(
-                    url + "/api/records/" + idv + "/draft",
-                    headers=headers,
-                )
-                if result.status_code != 201:
-                    raise Exception(result.text)
         # We want files to stay the same as the existing record
         data["files"] = existing["files"]
         if default_preview:

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api/caltechdata_write.py

@@ -67,45 +67,40 @@ def write_files_rdm(
                     raise Exception(result.text)
 
 
-def add_file_links(
-    metadata, file_links, file_descriptions=[], additional_descriptions="", s3_link=None
-):
+def add_file_links(file_upload_link, file_links, headers, keepfiles=False):
     # Currently configured for S3 links, assuming all are at the same endpoint
     link_string = ""
     endpoint = "https://" + file_links[0].split("/")[2]
     s3 = s3fs.S3FileSystem(anon=True, client_kwargs={"endpoint_url": endpoint})
     index = 0
+    file_json = []
+    f_list = []
     for link in file_links:
         file = link.split("/")[-1]
+        f_list.append(file)
         path = link.split(endpoint)[1]
         size = s3.info(path)["size"]
-        size = humanbytes(size)
-        try:
-            description = file_descriptions[index]
-            if description != " ":
-                desc = description + ","
-            else:
-                desc = ""
-        except IndexError:
-            desc = ""
-        if link_string == "":
-            if s3_link:
-                link_string = f"Files available via S3 at {s3_link}&lt;/p&gt;</p>"
-            else:
-                cleaned = link.strip(file)
-                link_string = f"Files available via S3 at {cleaned}&lt;/p&gt;</p>"
-        link_string += f"""{file}, {desc} {size}  
-        <p>&lt;a role="button" class="ui compact mini button" href="{link}"
-        &gt; &lt;i class="download icon"&gt;&lt;/i&gt; Download &lt;/a&gt;</p>&lt;/p&gt;</p>
-        """
-        index += 1
-    # Tack on any additional descriptions
-    if additional_descriptions != "":
-        link_string += additional_descriptions
 
-    description = {"description": link_string, "descriptionType": "files"}
-    metadata["descriptions"].append(description)
-    return metadata
+        file_json.append(
+            {"key": file, "size": size, "transfer": {"type": "R", "url": link}}
+        )
+    # Now we see if any existing files need to be replaced
+    result = requests.get(file_upload_link, headers=headers)
+    if result.status_code == 200:
+        ex_files = result.json()["entries"]
+        for ex in ex_files:
+            if ex["key"] in f_list:
+                result = requests.delete(ex["links"]["self"], headers=headers)
+                if result.status_code != 204:
+                    raise Exception(result.text)
+            else:
+                if keepfiles == False:
+                    result = requests.delete(ex["links"]["self"], headers=headers)
+                    if result.status_code != 204:
+                        raise Exception(result.text)
+    result = requests.post(file_upload_link, headers=headers, json=file_json)
+    if result.status_code != 201:
+        raise Exception(result.text)
 
 
 def send_to_community(
@@ -155,14 +150,13 @@ def caltechdata_write(
     token=None,
     files=[],
     production=False,
-    schema="43",
+    schema=None,
     publish=False,
     file_links=[],
     s3=None,
     community=None,
     authors=False,
     file_descriptions=[],
-    s3_link=None,
     default_preview=None,
     review_message=None,
     verify=True,
@@ -181,15 +175,25 @@ def caltechdata_write(
     if not token:
         token = os.environ["RDMTOK"]
 
+    if authors == False:
+        if production == True:
+            url = "https://data.caltech.edu/"
+        elif local == True:
+            url = "https://127.0.0.1:5000/"
+        else:
+            url = "https://data.caltechlibrary.dev/"
+    else:
+        if production == True:
+            url = "https://authors.library.caltech.edu/"
+        elif local == True:
+            url = "https://127.0.0.1:5000/"
+        else:
+            url = "https://authors.caltechlibrary.dev/"
+
     # If files is a string - change to single value array
     if isinstance(files, str) == True:
         files = [files]
 
-    if file_links:
-        metadata = add_file_links(
-            metadata, file_links, file_descriptions, s3_link=s3_link
-        )
-
     # Pull out pid information
     if production == True:
         repo_prefix = "10.22002"
@@ -235,22 +239,11 @@ def caltechdata_write(
     if "pids" not in metadata:
         metadata["pids"] = pids
 
-    if authors == False:
+    if authors == False and schema == "43":
         data = customize_schema.customize_schema(metadata, schema=schema)
-        if production == True:
-            url = "https://data.caltech.edu/"
-        elif local == True:
-            url = "https://127.0.0.1:5000/"
-        else:
-            url = "https://data.caltechlibrary.dev/"
     else:
+        # Using RDM schema, no customization
         data = metadata
-        if production == True:
-            url = "https://authors.library.caltech.edu/"
-        elif local == True:
-            url = "https://127.0.0.1:5000/"
-        else:
-            url = "https://authors.caltechlibrary.dev/"
 
     headers = {
         "Authorization": "Bearer %s" % token,
@@ -261,10 +254,13 @@ def caltechdata_write(
         "Content-type": "application/octet-stream",
     }
 
-    if not files:
+    if files or file_links:
+        if default_preview:
+            data["files"] = {"enabled": True, "default_preview": default_preview}
+        else:
+            data["files"] = {"enabled": True}
+    else:
         data["files"] = {"enabled": False}
-    elif default_preview:
-        data["files"] = {"enabled": True, "default_preview": default_preview}
 
     # Make draft and publish
     result = requests.post(
@@ -277,11 +273,14 @@ def caltechdata_write(
             raise Exception(result.text)
     idv = result.json()["id"]
     publish_link = result.json()["links"]["publish"]
+    file_link = result.json()["links"]["files"]
 
     if files:
-        file_link = result.json()["links"]["files"]
         write_files_rdm(files, file_link, headers, f_headers, s3, verify=verify)
 
+    if file_links:
+        add_file_links(file_link, file_links, headers)
+
     if community:
         review_link = result.json()["links"]["review"]
         send_to_community(

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api/cli.py

@@ -364,8 +364,8 @@ def upload_supporting_file(record_id=None):
             "Do you want to upload or link data files? (upload/link/n): "
         ).lower()
         if choice == "link":
-            endpoint = "sdsc.osn.xsede.org"
-            path = "ini230004-bucket01/"
+            endpoint = "caltech2.osn.mghpcc.org"
+            path = "caltechdata-public/"
             if not record_id:
                 write_s3cmd_config(endpoint)
                 print("""S3 connection configured.""")

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api/customize_schema.py

@@ -7,28 +7,6 @@ from pathlib import Path
 import requests
 
 
-def grid_to_ror(grid):
-    # Temporary until InvenioRDM stops spitting out GRIDS
-    # We manually handle some incorrect/redundant GRID Ids
-    if grid == "grid.451078.f":
-        ror = "00hm6j694"
-    elif grid == "grid.5805.8":
-        ror = "02en5vm52"
-    elif grid == "grid.465477.3":
-        ror = "00em52312"
-    else:
-        url = (
-            f"https://api.ror.org/organizations?query.advanced=external_ids.all:{grid}"
-        )
-        results = requests.get(url).json()
-        if len(results["items"]) == 0:
-            print(url + "doesn't have a valid ROR")
-            exit()
-        ror = results["items"][0]["id"]
-        ror = ror.split("ror.org/")[1]
-    return ror
-
-
 def get_vocabularies():
     """Returns dictionary of vocabularies"""
     path = Path(__file__).parent
@@ -297,7 +275,7 @@ def customize_schema_rdm(json_record):
             if "geoLocationPoint" in location:
                 lat = location["geoLocationPoint"]["pointLatitude"]
                 lon = location["geoLocationPoint"]["pointLongitude"]
-                new["geometry"] = {"type": "Point", "coordinates": [lat, lon]}
+                new["geometry"] = {"type": "Point", "coordinates": [lon, lat]}
             if "geoLocationBox" in location:
                 south = float(location["geoLocationBox"]["southBoundLatitude"])
                 north = float(location["geoLocationBox"]["northBoundLatitude"])
@@ -307,11 +285,11 @@ def customize_schema_rdm(json_record):
                     "type": "Polygon",
                     "coordinates": [
                         [
-                            [north, east],
-                            [north, west],
-                            [south, west],
-                            [south, east],
-                            [north, east],
+                            [east, north],
+                            [west, north],
+                            [west, south],
+                            [east, south],
+                            [east, north],
                         ]
                     ],
                 }
@@ -335,11 +313,6 @@ def customize_schema_rdm(json_record):
                         ror = ror.split("ror.org/")[1]
                     funder["id"] = ror
                     fund.pop("funderIdentifierType")
-                elif fund["funderIdentifierType"] == "GRID":
-                    # We need this temporarily to round-trip data
-                    ror = grid_to_ror(fund.pop("funderIdentifier"))
-                    funder["id"] = ror
-                    fund.pop("funderIdentifierType")
                 else:
                     print(f'Unknown Type mapping {fund["funderIdentifierType"]}')
             if "awardTitle" in fund:
@@ -568,8 +541,6 @@ def validate_metadata(json_record):
         else:
 
             for geo_loc in json_record["geoLocations"]:
-                if not isinstance(geo_loc, dict) or "geoLocationPlace" not in geo_loc:
-                    errors.append("Each 'geoLocation' must have 'geoLocationPlace'.")
                 if "geoLocationPoint" in geo_loc:
                     point = geo_loc["geoLocationPoint"]
                     if (

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api/get_metadata.py

@@ -12,11 +12,11 @@ def get_metadata(
     production=True,
     validate=True,
     emails=False,
-    schema="43",
+    schema=None,
     token=False,
     authors=False,
 ):
-    # Returns just DataCite metadata or DataCite metadata with emails
+    # Returns InvenioRDM metadata, or just DataCite metadata, or DataCite metadata with emails
 
     if production == True:
         if authors:
@@ -35,7 +35,7 @@ def get_metadata(
         "accept": "application/json",
     }
 
-    if authors:
+    if authors or schema == None:
         headers = base_headers
         validate = False
     else:
@@ -114,7 +114,7 @@ if __name__ == "__main__":
         action="store_true",
         help="skip validation of metadata",
     )
-    parser.add_argument("-schema", default="43", help="Schema Version")
+    parser.add_argument("-schema", default=None, help="Schema Version")
 
     args = parser.parse_args()
 

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/caltechdata_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caltechdata_api
-Version: 1.11.0
+Version: 2.0.0
 Summary: Python wrapper for CaltechDATA API.
 Home-page: https://github.com/caltechlibrary/caltechdata_api
 Author: Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/setup.cfg

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [metadata]
 name = caltechdata_api
-version = 1.11.0
+version = 2.0.0
 author = Thomas E Morrell, Rohan Bhattarai, Elizabeth Won, Alexander A Abakah, Kshemaahna Nagi
 author_email = tmorrell@caltech.edu, aabakah@caltech.edu, knagi@caltech.edu
 description = Python wrapper for CaltechDATA API.

{caltechdata_api-1.11.0 → caltechdata_api-2.0.0}/tests/test_rdm.py

@@ -68,8 +68,7 @@ def test_datacite_rdm_create_edit(full_datacite43_record):
 
     assert doi.startswith("10.33569")
 
-    # Validate is false until geolocation points are fixed/we move to 4.6
-    new_metadata = get_metadata(recid, production=False, validate=False)
+    new_metadata = get_metadata(recid, production=False, schema="43")
 
     assert new_metadata["publisher"] == "Edited"
 
@@ -89,7 +88,6 @@ def test_datacite_rdm_create_edit(full_datacite43_record):
 
     recid = new_doi.split("/")[1]
 
-    # Validate is false until geolocation points are fixed/we move to 4.6
-    new_metadata = get_metadata(recid, production=False, validate=False)
+    new_metadata = get_metadata(recid, production=False, schema="43")
 
     assert new_metadata["publisher"] == "Again!"