dsi-workflow 1.2.2__tar.gz → 1.2.3__tar.gz

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsi-workflow
-Version: 1.2.2
+Version: 1.2.3
 Summary: A Data Science Infrastructure Project
 Author-email: Jesus Pulido <pulido@lanl.gov>, James Ahrens <ahrens@lanl.gov>, Divya Banesh <dbanesh@lanl.gov>, Hugh Greenberg <hng@lanl.gov>, Pascal Grosset <pascalgrosset@lanl.gov>, Vedant Iyer <iyer@lanl.gov>, Benjamin Sims <bsims@lanl.gov>, Terece Turton <tlturton@lanl.gov>
 License-Expression: BSD-3-Clause

dsi_workflow-1.2.3/dsi/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.3"

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi/backends/oceans11.py

@@ -85,7 +85,7 @@ class Oceans11(Webserver):
         DEFAULT_URL = "https://oceans11.lanl.gov/dataCatalog/oceans11.db"
         base_url = url or DEFAULT_URL
 
-       # ----------------------------------------------------------------------
+        # ----------------------------------------------------------------------
         # Auth / connection config
         # ----------------------------------------------------------------------
         self.workspace = kwargs.get(
@@ -99,6 +99,10 @@ class Oceans11(Webserver):
 
         self.base_url = base_url.rstrip("/")
 
+        # skip data retrieval if only checking connection to oceans11
+        if kwargs.get("only_validate", False):
+            return
+
         # Data storage (tiered structure)
         # Tier 1: datasets, Tier 2: per-dataset resource tables
         self._cache = OrderedDict()
@@ -131,7 +135,7 @@ class Oceans11(Webserver):
     # ----------------------------------------------------------------------
     # Connection Validation
     # ----------------------------------------------------------------------
-    def validate_connection(self):
+    def validate_connection(self, **kwargs):
         """
         Validates that the base Oceans11 URL is accessible and functional.
         
@@ -162,15 +166,27 @@ class Oceans11(Webserver):
                 )
 
             if info is None:
+                if kwargs.get("only_validate", False):
+                    return False
                 raise ConnectionError(f"Failed to download catalog from {self.base_url}")
 
             local_path = info.get("local_path")
             if not local_path or not Path(local_path).is_file():
+                if kwargs.get("only_validate", False):
+                    return False
                 raise RuntimeError("Downloaded catalog file is invalid or missing")
 
+            # skip data retrieval if only checking connection to oceans11
+            if kwargs.get("only_validate", False):
+                import shutil
+                shutil.rmtree(info["folder_hash"] + "/")
+                return True
+
             return local_path
 
         except Exception as e:
+            if kwargs.get("only_validate", False):
+                return False
             raise ConnectionError(f"Unable to access Oceans11 catalog: {e}") from e
 
     # ---------------------------------------------------

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi/dsi.py

@@ -220,8 +220,8 @@ class DSI():
         n = OSTI()
         if n.validate_connection():
             print("OSTI : Read-only data catalog backend for discovering and querying OSTI (REST-based) open data resources.")
-        n = Oceans11()
-        if n.validate_connection():
+        n = Oceans11(only_validate=True)
+        if n.validate_connection(only_validate=True):
             print("Oceans11 : Read-only data catalog backend for discovering and querying Oceans11 (DSI-based) open data resources.")
         print()
 

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi/sync.py

@@ -12,6 +12,7 @@ from pathlib import Path
 from typing import Iterator
 from contextlib import redirect_stdout
 from collections import OrderedDict
+from urllib.parse import urlparse
 
 from dsi.core import Terminal
 from dsi.utils.federated.federate_datasets import federate_datasets, pull_data
@@ -778,7 +779,7 @@ class Sync():
             raise ValueError(f"Cannot download data from '{db_name}' because it has not been indexed by DSI.")
         remote_loc = t2.get_table("federated")["remote_location"].iloc[0]
 
-        with open(f"{db_data["workspace_folder"]}/host_usernames.json", "r", encoding="utf-8") as f:
+        with open(f'{db_data["workspace_folder"]}/host_usernames.json', "r", encoding="utf-8") as f:
             host_username = yaml.safe_load(f)
         username = (host_username or {}).get(db_data['location'], "")
 
@@ -787,13 +788,32 @@ class Sync():
         
         # add unique hash to download data
         workspace_folder = os.path.join(workspace_folder, db_data["folder_hash"])
-        
-        # Currently pulling all data referenced -- eventually allow user to download certain data
-        db_info, username = pull_data(db_data["location_type"], db_data["location"], remote_loc, 
-                                      workspace_folder, username, internal_use=True)
-        new_folder = Path(db_info.pop("new_db_folder"))
-        if new_folder.is_dir() and not any(new_folder.iterdir()):
-            new_folder.rmdir()
+
+        # if data files are URLs, then download them from filesystem, not using federated
+        def is_url(s):
+            try:
+                result = urlparse(s)
+                return all([result.scheme, result.netloc])
+            except Exception:
+                return False
+
+        if is_url(remote_loc):
+            remote_files = t2.get_table("filesystem")["file_remote"]
+            parent_url = os.path.commonprefix(remote_files.tolist())
+            for remote_url in remote_files:
+                # Downloading each file from fileystem
+                db_info, username = pull_data(db_data["location_type"], db_data["location"], remote_url, 
+                                            workspace_folder, username, internal_use=True, parent_hash=parent_url)
+                new_folder = Path(db_info.pop("new_db_folder"))
+                if new_folder.is_dir() and not any(new_folder.iterdir()):
+                    new_folder.rmdir()
+        else:        
+            # Currently pulling all referenced data -- eventually allow user to download certain data
+            db_info, username = pull_data(db_data["location_type"], db_data["location"], remote_loc, 
+                                        workspace_folder, username, internal_use=True)
+            new_folder = Path(db_info.pop("new_db_folder"))
+            if new_folder.is_dir() and not any(new_folder.iterdir()):
+                new_folder.rmdir()
 
 
     def gen_uuid(self, st):

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi/utils/federated/federate_datasets.py

@@ -79,7 +79,8 @@ def pull_data(location_type: str,
               abs_path_workspace_folder: str, 
               username: str,
               download_limit: int = 10485760,
-              internal_use = False) -> dict | tuple[dict | None, str]:
+              internal_use = False,
+              parent_hash: str = None) -> dict | tuple[dict | None, str]:
     """Pulls data from a specified location based on the location type (e.g., "github", "HPC", "HPC-Kerberos", "URL", "local"). 
     The function checks for existing files, compares them with remote versions using MD5 checksums, and downloads or skips files accordingly. 
     It also handles user interactions for confirming downloads of large files and manages host usernames for HPC access.
@@ -100,8 +101,11 @@ def pull_data(location_type: str,
     filename = get_last_part(path)
 
     # Create folder for data
-    abs_path_workspace_folder = str(Path(abs_path_workspace_folder).resolve()) 
-    folder_hash, abs_path_db_folder = create_folder_from_path(location, abs_path_workspace_folder)  
+    abs_path_workspace_folder = str(Path(abs_path_workspace_folder).resolve())
+    if parent_hash:
+        folder_hash, abs_path_db_folder = create_folder_from_path(parent_hash, abs_path_workspace_folder)
+    else:
+        folder_hash, abs_path_db_folder = create_folder_from_path(path, abs_path_workspace_folder)
 
     # Get the absolute path to the file to be downloaded
     file_path = Path(abs_path_db_folder) / filename

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi/utils/s3_utils.py

@@ -2,16 +2,18 @@ from pathlib import Path
 from urllib.parse import urlparse
 import getpass
 import os
-
-import boto3
-from botocore import UNSIGNED
-from botocore.config import Config
-from botocore.exceptions import (
-    BotoCoreError,
-    ClientError,
-    NoCredentialsError,
-    PartialCredentialsError,
-)
+import importlib.util
+
+if importlib.util.find_spec("duckdb") is not None:
+    import boto3
+    from botocore import UNSIGNED
+    from botocore.config import Config
+    from botocore.exceptions import (
+        BotoCoreError,
+        ClientError,
+        NoCredentialsError,
+        PartialCredentialsError,
+    )
 
 
 def get_s3_client(

{dsi_workflow-1.2.2 → dsi_workflow-1.2.3}/dsi_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsi-workflow
-Version: 1.2.2
+Version: 1.2.3
 Summary: A Data Science Infrastructure Project
 Author-email: Jesus Pulido <pulido@lanl.gov>, James Ahrens <ahrens@lanl.gov>, Divya Banesh <dbanesh@lanl.gov>, Hugh Greenberg <hng@lanl.gov>, Pascal Grosset <pascalgrosset@lanl.gov>, Vedant Iyer <iyer@lanl.gov>, Benjamin Sims <bsims@lanl.gov>, Terece Turton <tlturton@lanl.gov>
 License-Expression: BSD-3-Clause

dsi_workflow-1.2.2/dsi/_version.py

@@ -1 +0,0 @@
-__version__ = "1.2.2"