pysodafair 0.1.63__py3-none-any.whl → 0.1.64__py3-none-any.whl

pysoda/core/dataset_generation/upload.py

@@ -3830,57 +3830,39 @@ def generate_manifest_file_locally(generate_purpose, soda):
 
 
 def generate_manifest_file_data(dataset_structure):
-    # Define common file extensions with special handling
     double_extensions = {
         ".ome.tiff", ".ome.tif", ".ome.tf2", ".ome.tf8", ".ome.btf", ".ome.xml",
         ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz"
     }
 
-    # Helper function: Get the complete file extension
+    # Helper: Determine file extension (handles double extensions)
     def get_file_extension(filename):
         for ext in double_extensions:
             if filename.endswith(ext):
                 base_ext = os.path.splitext(os.path.splitext(filename)[0])[1]
                 return base_ext + ext
         return os.path.splitext(filename)[1]
-    
+
+    # Helper: Create a manifest row for a folder
     def create_folder_entry(folder_name, path_parts):
         full_path = "/".join(path_parts + [folder_name]) + "/"
-        entry = [
-            full_path.lstrip("/"),  # Remove leading slash for consistency
-            "", # Timestamp
-            "", # Description
-            "folder", # File type
-            "",  # Entity (empty)
-            "",  # Data modality (empty)
-            "",  # Also in dataset (empty)
-            "",  # Data dictionary path (empty)
-            "",  # Entity is transitive (empty)
-            "", # Additional Metadata
+        return [
+            full_path.lstrip("/"),
+            "", "", "folder", "", "", "", "", "", ""
         ]
-        return entry
-
-        
-
-    # Helper function: Build a single manifest entry
-    def create_file_entry(item, folder, path_parts, timestamp, filename):
-        full_path = "/".join(path_parts + [filename])
-        file_info = folder["files"][item]
 
+    # Helper: Create a manifest row for a file
+    def create_file_entry(file_name, file_info, path_parts, timestamp):
         entry = [
-            full_path.lstrip("/"),  # Remove leading slash for consistency
-            timestamp, # Timestamp
-            file_info["description"], # Description
-            get_file_extension(filename), # File type
-            "",  # Entity (empty)
-            "",  # Data modality (empty)
-            "",  # Also in dataset (empty)
-            "",  # Data dictionary path (empty)
-            "",  # Entity is transitive (empty)
-            file_info.get("additional-metadata", "") # Additional Metadata
+            "/".join(path_parts + [file_name]).lstrip("/"),
+            timestamp,
+            file_info["description"],
+            get_file_extension(file_name),
+            "", "", "", "", "",
+            file_info.get("additional-metadata", "")
         ]
 
-        # Add any extra columns dynamically
+        # Append any extra columns dynamically
         if "extra_columns" in file_info:
             for key, value in file_info["extra_columns"].items():
                 entry.append(value)
@@ -3889,48 +3871,36 @@ def generate_manifest_file_data(dataset_structure):
 
         return entry
 
-    # Recursive function: Traverse dataset and collect file data
+    # Recursive traversal of folders and files
     def traverse_folders(folder, path_parts):
-        # Add header row if processing files for the first time
         if not manifest_data:
             manifest_data.append(header_row)
-        
-        if "files" in folder:
-            for item, file_info in folder["files"].items():
-
-                if "path" in file_info:
-                    file_path = file_info["path"]
-                elif "pspath" in file_info:
-                    file_path = file_info["pspath"]
-                else: 
-                    continue 
-
-                # If the file is a manifest file, skip it
-                if item in {"manifest.xlsx", "manifest.csv"}:
-                    continue
 
-                # Determine timestamp 
-                filename = os.path.basename(file_path.replace("\\", "/"))
-                if file_info["location"] == "ps":
-                    timestamp = file_info["timestamp"]
-                else:
-                    local_path = pathlib.Path(file_info["path"])
-                    timestamp = datetime.fromtimestamp(
-                        local_path.stat().st_mtime, tz=local_timezone
-                    ).isoformat().replace(".", ",").replace("+00:00", "Z")
-
-                # Add file entry
-                manifest_data.append(create_file_entry(item, folder, path_parts, timestamp, filename))
-
-        if "folders" in folder:
-            for subfolder_name, subfolder in folder["folders"].items():
-                # Add folder entry
-                manifest_data.append(create_folder_entry(subfolder_name, path_parts))
-                traverse_folders(subfolder, path_parts + [subfolder_name])
-
-    # Initialize variables
-    manifest_data = []  # Collects all rows for the manifest
-    # TODO: Update to SDS 3.0
+        # Process files
+        for file_name, file_info in folder.get("files", {}).items():
+            file_path = file_info.get("path")
+            if not file_path:
+                continue
+            if file_name in {"manifest.xlsx", "manifest.csv"}:
+                continue
+
+            if file_info["location"] == "ps":
+                timestamp = file_info["timestamp"]
+            else:
+                local_path = pathlib.Path(file_info["path"])
+                timestamp = datetime.fromtimestamp(
+                    local_path.stat().st_mtime, tz=local_timezone
+                ).isoformat().replace(".", ",").replace("+00:00", "Z")
+
+            manifest_data.append(create_file_entry(file_name, file_info, path_parts, timestamp))
+
+        # Process subfolders
+        for subfolder_name, subfolder in folder.get("folders", {}).items():
+            manifest_data.append(create_folder_entry(subfolder_name, path_parts))
+            traverse_folders(subfolder, path_parts + [subfolder_name])
+
+    # Initialize manifest data and header
+    manifest_data = []
     header_row = [
         "filename", "timestamp", "description", "file type", "entity",
         "data modality", "also in dataset", "data dictionary path",
@@ -3938,9 +3908,6 @@ def generate_manifest_file_data(dataset_structure):
     ]
     local_timezone = TZLOCAL()
 
-    # Log the dataset structure
-
-    # Start recursive traversal from the root
     traverse_folders(dataset_structure, [])
 
     return manifest_data
@@ -3948,4 +3915,3 @@ def generate_manifest_file_data(dataset_structure):
 
 
 
-

{pysodafair-0.1.63.dist-info → pysodafair-0.1.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pysodafair
-Version: 0.1.63
+Version: 0.1.64
 Summary: Pysoda package for Fairdataihub tools
 License: MIT
 License-File: LICENSE

{pysodafair-0.1.63.dist-info → pysodafair-0.1.64.dist-info}/RECORD

@@ -4,7 +4,7 @@ pysoda/core/__init__.py,sha256=bXnu4fYemJ915xId8nwh-Gy2IUEvVuoS9Hk3CXyJp8U,235
 pysoda/core/dataset_generation/__init__.py,sha256=tisLmJeXLeAINXf7BdNZGi2i9vQwImWGa_S5FNwQFbs,299
 pysoda/core/dataset_generation/manifestSession/__init__.py,sha256=kqkTAFEhluyQJ9mmMhYQTh1tgowBeJdH-Q9s5ifGkSE,51
 pysoda/core/dataset_generation/manifestSession/manifest_session.py,sha256=TML_KOJ-1REohqSaHCZNFJrbDR1UDQ9sFcithJRx9t8,4669
-pysoda/core/dataset_generation/upload.py,sha256=ATx8D-Ck1xDzrT_rI-YSDmi9hFFqiz2E0FgCrPlyJQA,174022
+pysoda/core/dataset_generation/upload.py,sha256=FtEQoyg3ly0lXmnUtKDXXHln1dLN6reb9knOdFTBFeA,172657
 pysoda/core/dataset_importing/__init__.py,sha256=NbWs4HAqqydFLACFoVwl6g457dkYvjiZKx9NzB9wFxE,114
 pysoda/core/dataset_importing/import_dataset.py,sha256=cx8qCQmR_BKdC2G-jzqE4dWg2JhkOS3jM8kpjLIsObk,29487
 pysoda/core/metadata/__init__.py,sha256=Tkx6vdEQEPwAHmVSc9GfdbDZUkvuAqEgHJOxRDeX5vE,821
@@ -71,7 +71,7 @@ pysoda/utils/profile.py,sha256=di4D_IE1rGSfHl0-SRVjrJK2bCdedW4ugp0W-j1HarQ,937
 pysoda/utils/schema_validation.py,sha256=3w3FRPyn4P3xMhITzItk-jYte8TTlNrgCWmY-s05x9Y,4438
 pysoda/utils/time_utils.py,sha256=g5848bivtzWj6TDoWo6CcohF-THxShETP4qTyHTjBWw,131
 pysoda/utils/upload_utils.py,sha256=-BCvfXsJSFqnwEZVCFQX1PLfkdeW0gyGLjLGW6Uxdf8,4607
-pysodafair-0.1.63.dist-info/METADATA,sha256=i8T8w0remPrPYYbifUYbopeoVuhljwmxZv1HGVdoAMc,7011
-pysodafair-0.1.63.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pysodafair-0.1.63.dist-info/licenses/LICENSE,sha256=Jlt0uGnx87qPRGXPQHsBkg_S7MOsYS2E9Rh1zy47bfw,1082
-pysodafair-0.1.63.dist-info/RECORD,,
+pysodafair-0.1.64.dist-info/METADATA,sha256=mgUEcj-8RZdfFlMBD94Joq0bmRP4CXOF0C6S3JxYX20,7011
+pysodafair-0.1.64.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pysodafair-0.1.64.dist-info/licenses/LICENSE,sha256=Jlt0uGnx87qPRGXPQHsBkg_S7MOsYS2E9Rh1zy47bfw,1082
+pysodafair-0.1.64.dist-info/RECORD,,