pysodafair 0.1.64__py3-none-any.whl → 0.1.65__py3-none-any.whl

pysoda/core/dataset_generation/upload.py

@@ -40,7 +40,7 @@ from os.path import (
 import pandas as pd
 import time
 from timeit import default_timer as timer
-from datetime import timedelta
+from datetime import timedelta, timezone
 import shutil
 import subprocess
 import gevent
@@ -3888,9 +3888,11 @@ def generate_manifest_file_data(dataset_structure):
                 timestamp = file_info["timestamp"]
             else:
                 local_path = pathlib.Path(file_info["path"])
-                timestamp = datetime.fromtimestamp(
-                    local_path.stat().st_mtime, tz=local_timezone
-                ).isoformat().replace(".", ",").replace("+00:00", "Z")
+                # Create proper ISO 8601 timestamp
+                dt = datetime.fromtimestamp(local_path.stat().st_mtime, tz=timezone.utc)
+                # per the SDS spec, replace '.' with ',' in the timestamp fractional seconds section
+                timestamp = dt.isoformat().replace(".", ",").replace("+00:00", "Z")
+
 
             manifest_data.append(create_file_entry(file_name, file_info, path_parts, timestamp))
 

pysoda/core/metadata/dataset_description.py

@@ -42,21 +42,23 @@ def create_excel(
         .get("dataset_type", "")
     )
 
-    populate_standards_info(ws1, soda)
+    standards_arr_len = populate_standards_info(ws1, soda)
 
-    keyword_array_len = populate_basic_info(ws1, soda)
+    keyword_funding_array_len = populate_basic_info(ws1, soda)
     study_arr_len = populate_study_info(ws1, soda)
     contributor_arr_len = populate_contributor_info(ws1, soda)
     related_resource_arr_len = populate_related_resource_information(ws1, soda)
     populate_funding_info(ws1, soda)
     populate_participant_information(ws1, soda)
-    data_dictionary_information(ws1, soda)
+    dict_arr_len = data_dictionary_information(ws1, soda)
 
     max_len = max(
-        keyword_array_len,
+        keyword_funding_array_len,
         study_arr_len,
         contributor_arr_len,
         related_resource_arr_len,
+        standards_arr_len,
+        dict_arr_len,
     )
 
     # 3 is the first value column position
@@ -102,9 +104,12 @@ def populate_study_info(workbook, soda):
 
 def populate_standards_info(workbook, soda):
     standards_info = soda["dataset_metadata"]["dataset_description"]["standards_information"]
-    workbook["D5"] = standards_info["data_standard"]
-    workbook["D6"] = standards_info["data_standard_version"]
+    # this is an array with multiple entries
+    for col, standard in zip(excel_columns(start_index=3), standards_info):
+        workbook[col + "5"] = standard.get("data_standard", "")
+        workbook[col + "6"] = standard.get("data_standard_version", "")
 
+    return max(1, len(standards_info))
 
 def populate_basic_info(workbook, soda):
     basic_info = soda["dataset_metadata"]["dataset_description"]["basic_information"]
@@ -117,12 +122,14 @@ def populate_basic_info(workbook, soda):
     for col, keyword in zip(excel_columns(start_index=3), keywords):
         workbook[f"{col}11"] = keyword
 
-    workbook["D12"] = basic_info.get("funding", "")
+    funding = basic_info.get("funding", [])
+    for col, funding_source in zip(excel_columns(start_index=3), funding):
+        workbook[f"{col}12"] = funding_source
     workbook["D13"] = basic_info.get("acknowledgments", "")
     workbook["D14"] = basic_info.get("license", "")
 
     # Return the length of the keywords array, or 1 if empty
-    return max(1, len(keywords))
+    return max(1, len(keywords), len(funding))
 
 
 def populate_funding_info(workbook, soda):
@@ -170,11 +177,14 @@ def data_dictionary_information(workbook, soda):
     It currently does not populate any data in the workbook.
     """
     # Placeholder for future implementation
-    data_dictionary_info = soda["dataset_metadata"]["dataset_description"].get("data_dictionary_information", {})
+    data_dictionary_info = soda["dataset_metadata"]["dataset_description"].get("data_dictionary_information", [])
 
-    workbook["D43"] = data_dictionary_info.get("data_dictionary_path", "")
-    workbook["D44"] = data_dictionary_info.get("data_dictionary_type", "")
-    workbook["D45"] = data_dictionary_info.get("data_dictionary_description", "")
+    for column, entry in zip(excel_columns(start_index=3), data_dictionary_info):
+        workbook[column + "43"] = entry.get("data_dictionary_path", "")
+        workbook[column + "44"] = entry.get("data_dictionary_type", "")
+        workbook[column + "45"] = entry.get("data_dictionary_description", "")
+
+    return max(1, len(data_dictionary_info))
 
 def grayout_subheaders(workbook, col):
     """
@@ -228,9 +238,6 @@ def apply_dashed_border(cell, workbook):
     
 
 
-
-
-
 def extend_value_header(workbook, max_len, start_index):
     """
     The headers starting at G1 are the 'Value' headers that correspond to the maximum number of entries for either the 
@@ -253,4 +260,4 @@ def extend_value_header(workbook, max_len, start_index):
         apply_calibri_bold_12(header_cell)
         set_cell_alignment(header_cell, horizontal='center', vertical='center')
         apply_dashed_border(header_cell, workbook)
-        grayout_subheaders(workbook, column_list[i - 1])
+        grayout_subheaders(workbook, column_list[i - 1])

pysoda/core/metadata/manifest.py

@@ -21,7 +21,7 @@ def create_excel(soda, upload_boolean, local_destination):
     wb = load_workbook(destination)
     ws1 = wb["Sheet1"]
     manifest = soda["dataset_metadata"]["manifest_file"]
-    # validate_schema(manifest, SCHEMA_NAME_MANIFEST)
+    validate_schema(manifest, SCHEMA_NAME_MANIFEST)
     ascii_headers = excel_columns(start_index=0)
     custom_headers_to_column = {}
 

pysoda/schema/dataset_description.json

@@ -8,21 +8,25 @@
     },
     "type": {
       "type": "string",
-      "description": "The type of the dataset. For example, experimental."
+      "enum": ["experimental", "computational"],
+      "description": "The type of the dataset. In short, experimental data should have data collected from subjects and/or samples."
     },
     "standards_information": {
-      "type": "object",
-      "properties": {
-        "data_standard": {
-          "type": "string",
-          "description": "The name of the standard used in the project."
-        },
-        "data_standard_version": {
-          "type": "string",
-          "description": "The version of the standard used in the project."
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "properties": {
+          "data_standard": {
+            "type": "string",
+            "description": "The name of the standard used in the project. For example, SPARC."
+          },
+          "data_standard_version": {
+            "type": "string",
+            "description": "The version of the standard used in the project. For example, 1.0.0"
+          }
         }
-      },
-      "required": []
+      }
     },
     "basic_information": {
       "type": "object",
@@ -47,8 +51,11 @@
           "description": "A list of keywords related to the project."
         },
         "funding": {
-          "type": "string",
-          "description": "Funding awards for the project."
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Funding awards for the project. Listed as free text. E.g., OT2OD025349"
         },
         "acknowledgments": {
           "type": "string",
@@ -56,7 +63,7 @@
         },
         "license": {
           "type": "string",
-          "description": "The license under which the project is released."
+          "description": "The license under which the project is released. Use the SPDX license identifier."
         }
       },
       "required": []
@@ -128,10 +135,12 @@
         "properties": {
           "contributor_orcid_id": {
             "type": "string",
-            "description": "The ORCiD for this contributor."
+            "pattern": "^https://orcid.org/000[09]-00[01][0-9]-[0-9]{4}-[0-9]{3}([0-9]|X)$",
+            "description": "The ORCiD for this contributor. Must be a valid ORCID URL format."
           },
           "contributor_affiliation": {
             "type": "string",
+            "pattern": "^https://ror.org/0[0-9a-z]{6}[0-9]{2}$",
             "description": "The institutional affiliation for this contributor."
           },
           "contributor_name": {
@@ -142,22 +151,27 @@
           "contributor_role": {
             "type": "string",
             "enum": [
-              "PrincipalInvestigator",
-              "Creator",
-              "CoInvestigator",
               "CorrespondingAuthor",
+              "ContactPerson",
+              "Creator",
               "DataCollector",
               "DataCurator",
               "DataManager",
               "Distributor",
               "Editor",
+              "HostingInstitution",
+              "PrincipalInvestigator",
+              "CoInvestigator",
               "Producer",
               "ProjectLeader",
               "ProjectManager",
               "ProjectMember",
+              "RegistrationAgency",
+              "RegistrationAuthority",
               "RelatedPerson",
               "Researcher",
               "ResearchGroup",
+              "RightsHolder",
               "Sponsor",
               "Supervisor",
               "WorkPackageLeader",
@@ -256,40 +270,48 @@
       "type": "object",
       "properties": {
         "number_of_subjects": {
-          "type": "number",
+          "type": "integer",
+          "minimum": 0,
           "description": "The number of subjects in the study."
         },
         "number_of_samples": {
-          "type": "number",
+          "type": "integer",
+          "minimum": 0,
           "description": "The number of samples in the study."
         },
         "number_of_sites": {
-          "type": "number",
+          "type": "integer",
+          "minimum": 0,
           "description": "The number of sites in the study."
         },
         "number_of_performances": {
-          "type": "number",
+          "type": "integer",
+          "minimum": 0,
           "description": "The number of performance in the study."
         }
       },
       "required": []
     },
     "data_dictionary_information": {
-      "type": "object",
-      "properties": {
-        "data_dictionary_path": {
-          "type": "string",
-          "description": "The path to the data dictionary file."
-        },
-        "data_dictionary_type": {
-          "type": "string",
-          "description": "The type of the data dictionary. E.g., json-schema"
-        },
-        "data_dictionary_description": {
-          "type": "string",
-          "description": "A descrption of the data dictionary."
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "data_dictionary_path": {
+            "type": "string",
+            "description": "The path to the data dictionary file. Should be relative to the root of the dataset. E.g., 'code/data_dictionary/schema.json'"
+          },
+          "data_dictionary_type": {
+            "type": "string",
+            "description": "The type of the data dictionary. E.g., json-schema"
+          },
+          "data_dictionary_description": {
+            "type": "string",
+            "description": "A description of the data dictionary."
+          }
         }
       }
     }
-  }
+  },
+  "required": ["metadata_version"]
 }

pysoda/schema/manifest.json

@@ -11,7 +11,8 @@
         },
         "timestamp": {
           "type": "string",
-          "description": "Timestamp of when the data was created or last modified. This should be in ISO 8601 format."
+          "description": "Timestamp of when the data was created or last modified. This should be in ISO 8601 format. Per the SDS guidelines, commas are to be used for separating fractional seconds.",
+          "pattern": "^[0-9]{4}-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9](:[0-6][0-9](,[0-9]{1,9})?)?(Z|[+-][0-2][0-9]:[0-6][0-9])$"
         },
         "description": {
           "type": "string",
@@ -23,7 +24,8 @@
         },
         "entity": {
           "type": "string",
-          "description": "Each ID should be taken from the subject.xlsx, samples.xlsx, sites.xlsx, or performances.xlsx files."
+          "description": "Each ID should be taken from the subject.xlsx, samples.xlsx, sites.xlsx, or performances.xlsx files.",
+          "pattern": "^((pop-)?(sub|sam|site|perf)-[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?)?$"
         },
         "data_modality": {
           "type": "string",
@@ -43,7 +45,7 @@
         },
         "entity_is_transitive": {
           "type": "string",
-          "description": "Indicates whether the entity represented in this data file is transitive. This should be either true or false."
+          "description": "Indicates whether the entity represented in this data file is transitive. This should be either true or false if provided ."
         },
         "additional_metadata": {
           "type": "string",

{pysodafair-0.1.64.dist-info → pysodafair-0.1.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pysodafair
-Version: 0.1.64
+Version: 0.1.65
 Summary: Pysoda package for Fairdataihub tools
 License: MIT
 License-File: LICENSE

{pysodafair-0.1.64.dist-info → pysodafair-0.1.65.dist-info}/RECORD

@@ -4,16 +4,16 @@ pysoda/core/__init__.py,sha256=bXnu4fYemJ915xId8nwh-Gy2IUEvVuoS9Hk3CXyJp8U,235
 pysoda/core/dataset_generation/__init__.py,sha256=tisLmJeXLeAINXf7BdNZGi2i9vQwImWGa_S5FNwQFbs,299
 pysoda/core/dataset_generation/manifestSession/__init__.py,sha256=kqkTAFEhluyQJ9mmMhYQTh1tgowBeJdH-Q9s5ifGkSE,51
 pysoda/core/dataset_generation/manifestSession/manifest_session.py,sha256=TML_KOJ-1REohqSaHCZNFJrbDR1UDQ9sFcithJRx9t8,4669
-pysoda/core/dataset_generation/upload.py,sha256=FtEQoyg3ly0lXmnUtKDXXHln1dLN6reb9knOdFTBFeA,172657
+pysoda/core/dataset_generation/upload.py,sha256=lnU6od0LtJOjuzNj-VX2HzOp66xUBr1V6nXtpJjkdIg,172804
 pysoda/core/dataset_importing/__init__.py,sha256=NbWs4HAqqydFLACFoVwl6g457dkYvjiZKx9NzB9wFxE,114
 pysoda/core/dataset_importing/import_dataset.py,sha256=cx8qCQmR_BKdC2G-jzqE4dWg2JhkOS3jM8kpjLIsObk,29487
 pysoda/core/metadata/__init__.py,sha256=Tkx6vdEQEPwAHmVSc9GfdbDZUkvuAqEgHJOxRDeX5vE,821
 pysoda/core/metadata/code_description.py,sha256=sWbRPWPf32txpN048oQ9lT3y7GAoDkrS-GHivwwv55Y,4104
 pysoda/core/metadata/constants.py,sha256=PR78huqBKdBpzDUGxVKy9YW3pUrJ4ftuF4MfqpNb1bU,1052
-pysoda/core/metadata/dataset_description.py,sha256=K1yWYQkbn4d7DCjQH5vT9Y6dhcWWlx823XLACgkIS2M,9749
+pysoda/core/metadata/dataset_description.py,sha256=icIRH2TYXWqrc4gu_RUK0yP_WQmy9BGAkVOEVQ--GfA,10275
 pysoda/core/metadata/excel_utils.py,sha256=FQ8-DBq2lxFdpDUZeABfFKe284JzRTe6AYPxEw7yzsg,1216
 pysoda/core/metadata/helpers.py,sha256=jDf4KPTbx4unHT7pDlFEa0jls1OZY-dpdKf3kUGMtvg,10609
-pysoda/core/metadata/manifest.py,sha256=MjID-r52Yn3i9WlrT5IWeYvmkVdfzQuqGOmoMsZQVJg,4197
+pysoda/core/metadata/manifest.py,sha256=LO8ZP8oJ0vEjbXU0yPVEYm62MLOZwIYVUaW4VEczO9k,4195
 pysoda/core/metadata/manifest_package/__init__.py,sha256=7qiNT62WsI1LKb-lvnK31lW04sOvW_KTZQA1e8sUAgY,211
 pysoda/core/metadata/manifest_package/manifest.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pysoda/core/metadata/manifest_package/manifest_import.py,sha256=5rKOBf-NMt2dVsPJYotk5V855wnmtiMqrpknSOYOEl0,1347
@@ -50,8 +50,8 @@ pysoda/core/upload_manifests/__init__.py,sha256=SOiyflYmZDtkLwWpeu7flyn3J2SAj9WL
 pysoda/core/upload_manifests/upload_manifests.py,sha256=Hd2XSXYieG-EMcPTLSYXvdtraipp1XBSo5cGpZR-vbA,1223
 pysoda/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pysoda/schema/code_description.json,sha256=FJOyJBC2TKjTTdtew3B4wdq_WIrti-r2F-M6t5OVNB0,17854
-pysoda/schema/dataset_description.json,sha256=qxhzQzoE7CGuXvtY9fVWV31-daf5Ivms7_PkVW4i858,9053
-pysoda/schema/manifest.json,sha256=bsVzhnZHd2566KhrxQPBcwH376fgE8pSY8eXviTXrWA,2636
+pysoda/schema/dataset_description.json,sha256=U725xGDmy4oeJe0IOFcJc7a0oA5pAbpsRVQsjhPEj1k,10069
+pysoda/schema/manifest.json,sha256=KKi7_W5yG-WGhZwLqMER_sfjUgE3056NGR2Hj-Ljzsk,2961
 pysoda/schema/performances.json,sha256=TJw7ERC6eMR-9cImOg7xoEVRyOR4xViJqZO8U2X2OXo,1254
 pysoda/schema/resources.json,sha256=9-rwCTX523V5XaKmXLTUGzackOl7tk-qijbbBDoeuhY,1232
 pysoda/schema/samples.json,sha256=NvuYhskV9QFrkSX4ant9G4KFCe9JLnVG98vDv2Puuvk,3340
@@ -71,7 +71,7 @@ pysoda/utils/profile.py,sha256=di4D_IE1rGSfHl0-SRVjrJK2bCdedW4ugp0W-j1HarQ,937
 pysoda/utils/schema_validation.py,sha256=3w3FRPyn4P3xMhITzItk-jYte8TTlNrgCWmY-s05x9Y,4438
 pysoda/utils/time_utils.py,sha256=g5848bivtzWj6TDoWo6CcohF-THxShETP4qTyHTjBWw,131
 pysoda/utils/upload_utils.py,sha256=-BCvfXsJSFqnwEZVCFQX1PLfkdeW0gyGLjLGW6Uxdf8,4607
-pysodafair-0.1.64.dist-info/METADATA,sha256=mgUEcj-8RZdfFlMBD94Joq0bmRP4CXOF0C6S3JxYX20,7011
-pysodafair-0.1.64.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pysodafair-0.1.64.dist-info/licenses/LICENSE,sha256=Jlt0uGnx87qPRGXPQHsBkg_S7MOsYS2E9Rh1zy47bfw,1082
-pysodafair-0.1.64.dist-info/RECORD,,
+pysodafair-0.1.65.dist-info/METADATA,sha256=R7lpbhjF4h0BHtuvf1_nfy8aEINhAFfBChBklci6G8I,7011
+pysodafair-0.1.65.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pysodafair-0.1.65.dist-info/licenses/LICENSE,sha256=Jlt0uGnx87qPRGXPQHsBkg_S7MOsYS2E9Rh1zy47bfw,1082
+pysodafair-0.1.65.dist-info/RECORD,,