dsi-workflow 1.2.2__tar.gz → 1.2.4__tar.gz

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsi-workflow
-Version: 1.2.2
+Version: 1.2.4
 Summary: A Data Science Infrastructure Project
 Author-email: Jesus Pulido <pulido@lanl.gov>, James Ahrens <ahrens@lanl.gov>, Divya Banesh <dbanesh@lanl.gov>, Hugh Greenberg <hng@lanl.gov>, Pascal Grosset <pascalgrosset@lanl.gov>, Vedant Iyer <iyer@lanl.gov>, Benjamin Sims <bsims@lanl.gov>, Terece Turton <tlturton@lanl.gov>
 License-Expression: BSD-3-Clause
@@ -12,7 +12,7 @@ Requires-Dist: pydantic>=2.1.1
 Requires-Dist: nbconvert>=7.13.0
 Requires-Dist: matplotlib>=3.6.0
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: requests>=2.31.0
+Requires-Dist: requests>=2.32.0
 Requires-Dist: pyreadline3; platform_system == "Windows"
 
 ===============

dsi_workflow-1.2.4/dsi/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.4"

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/dsi/backends/backend.py

@@ -18,14 +18,6 @@ class Backend(ABC):
     def get_table(self, table_name, **kwargs):
         pass
 
-    @abstractmethod
-    def get_table_names(self,query):
-        pass
-
-    @abstractmethod
-    def notebook(self, **kwargs):
-        pass
-
     @abstractmethod
     def process_artifacts(self, **kwargs):
         pass

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/dsi/backends/duckdb.py

@@ -329,8 +329,8 @@ class DuckDB(Filesystem):
             
             self.ingest_table_helper(types, foreign_query)
             
-            # TODO: move this check to schema reader by allowing users to just create table without data
-            if not all(v == [""] for v in tableData.values()): # if table is just one row of empty strings, don't insert
+            # skip insert if table is empty
+            if not all(v in ([], [""], [None]) for v in tableData.values()): 
                 col_names = ', '.join(types.properties.keys())
                 placeholders = ', '.join('?' * len(types.properties))
 
@@ -438,16 +438,19 @@ class DuckDB(Filesystem):
                 raise
         else:
             raise RuntimeError("Can only run SELECT, PRAGMA, UPDATE, or ALTER queries on the data")
-        
+
+        primary_table = self.get_sql_query_table_name(query)
+        if primary_table is None:
+                    raise ValueError("Query must reference at least one table")
+
         if dict_return:
-            tables = self.get_table_names(query)
-            if len(tables) > 1:
-                raise RuntimeError("Can only return ordered dictionary if querying one table")
-            
             return OrderedDict(data.to_dict(orient='list'))
         else:
+            data.attrs["table_name"] = primary_table
             return data
-    
+
+
+
     def get_table(self, table_name, dict_return = False):
         """
         Retrieves all data from a specified table without requiring knowledge of SQL.
@@ -466,8 +469,10 @@ class DuckDB(Filesystem):
             - If `dict_return` is True: returns an OrderedDict
         """
         return self.query_artifacts(query=f"SELECT * FROM {table_name}", dict_return=dict_return)
-    
-    def get_table_names(self, query):
+
+
+
+    def get_sql_query_table_name(self, query):
         """
         Extracts all table names from a SQL query. Helper function for `query_artifacts()` that users do not need to call
 
@@ -477,10 +482,20 @@ class DuckDB(Filesystem):
         Return: list of str
             List of table names referenced in the query.
         """
-        all_names = re.findall(r'FROM\s+["\']?([\w\-]+)["\']?|JOIN\s+["\']?([\w\-]+)["\']?', query, re.IGNORECASE)
-        tables = [table for from_tbl, join_tbl in all_names if (table := from_tbl or join_tbl)]
-        return tables
-    
+        try:
+            from sqlglot import parse_one, exp
+            tree = parse_one(query, dialect="duckdb")
+            cte_names = {cte.alias_or_name for cte in tree.find_all(exp.CTE)}
+            query_tables = [table.name for table in tree.find_all(exp.Table) if table.name not in cte_names]
+
+        except ModuleNotFoundError:
+            all_names = re.findall(r'FROM\s+["\']?([\w\-]+)["\']?|JOIN\s+["\']?([\w\-]+)["\']?', query, re.IGNORECASE)
+            query_tables = [table for from_tbl, join_tbl in all_names if (table := from_tbl or join_tbl)]
+
+        return query_tables[0] if query_tables else None
+
+
+
     def get_schema(self):
         """
         Returns the structural schema of this database in the form of CREATE TABLE statements.
@@ -490,12 +505,15 @@ class DuckDB(Filesystem):
         """
         schema_stmts = self.query_artifacts(query="SELECT sql FROM duckdb_tables where sql NOT NULL ")
         return schema_stmts["sql"].str.cat(sep="\n")
-    
-    
+
+
+
     def notebook(self, interactive=False):
-        pass
+        """Notebook generation is not supported for DuckDB backend"""
+        raise NotImplementedError("Notebook generation not supported for DuckDB backend")
+
+
 
-    
     def process_artifacts(self, only_units_relations = False):
         """
         Reads data from the DuckDB database into a nested OrderedDict.

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/dsi/backends/filesystem.py

@@ -19,14 +19,6 @@ class Filesystem(Backend, ABC):
     def get_table(self, table_name, **kwargs):
         pass
 
-    @abstractmethod
-    def get_table_names(self,query):
-        pass
-
-    @abstractmethod
-    def notebook(self, **kwargs):
-        pass
-
     @abstractmethod
     def process_artifacts(self, **kwargs):
         pass

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/dsi/backends/gufi.py

@@ -91,8 +91,8 @@ class Gufi(Filesystem):
             con.enable_load_extension(False)
 
             dsi_column_names = ",".join(self.dsi_columns)
-            gufi_column_names = ",".join((["rpath(sname, sroll, name) AS fullpath"] + self.gufi_columns[1:]
-                                    if self.gufi_columns[0] == "fullpath" else self.gufi_columns))
+            gufi_column_names = ",".join(["rpath(sname, sroll, name) AS fullpath"] + self.gufi_columns[1:]
+                                    if self.gufi_columns[0] == "fullpath" else self.gufi_columns)
             query=f"""
             CREATE VIRTUAL TABLE uview USING gufi_vt(
                 threads=64,
@@ -144,8 +144,6 @@ class Gufi(Filesystem):
         pass
     def get_table(self):
         raise NotImplementedError("Cannot get table data with the GUFI backend")
-    def get_table_names(self):
-        raise NotImplementedError("Cannot get table names with the GUFI backend")
     def list(self):
         raise NotImplementedError("Cannot list tables with the GUFI backend")
     def notebook(self):

{dsi_workflow-1.2.2 → dsi_workflow-1.2.4}/dsi/backends/hpss.py

@@ -50,7 +50,7 @@ class HPSS(Backend):
             os.chdir(new_dir)
             file_to_put = local_file.split('/')[-1]
 
-        stdout, stderr, returncode = self.run_hsi("put", [file_to_put])
+        _, _, returncode = self.run_hsi("put", [file_to_put])
         if new_dir is not None:
             os.chdir(cwd)
 
@@ -71,7 +71,7 @@ class HPSS(Backend):
             print("Error changing to temp dir: %s" % tmp_dir)
             return False
 
-        stdout, stderr, returncode = self.run_hsi("get", hpss_file)
+        _, _, returncode = self.run_hsi("get", hpss_file)
         try:
             os.chdir(cwd)
         except Exception: