dsi-workflow 1.0__py3-none-any.whl

dsi/_version.py

@@ -0,0 +1 @@
+__version__ = "1.0"

dsi/backends/filesystem.py

@@ -0,0 +1,55 @@
+from abc import ABCMeta, abstractmethod
+
+
+class Backend(metaclass=ABCMeta):
+    @abstractmethod
+    def __init__(self, filename) -> None:
+        pass
+
+    @property
+    @abstractmethod
+    def git_commit_sha(self):
+        pass
+
+    @abstractmethod
+    def put_artifacts(self, artifacts, kwargs) -> None:
+        pass
+
+    @abstractmethod
+    def get_artifacts(self, query):
+        pass
+
+    @abstractmethod
+    def inspect_artifacts(self):
+        pass
+
+
+class Filesystem(Backend):
+    git_commit_sha = '5d79e08d4a6c1570ceb47cdd61d2259505c05de9'
+    # Declare named types
+    DOUBLE = "DOUBLE"
+    STRING = "VARCHAR"
+    FLOAT = "FLOAT"
+    INT = "INT"
+
+    # Declare store types
+    GUFI_STORE = "gufi"
+    SQLITE_STORE = "sqlite"
+    PARQUET_STORE = "parquet"
+
+    # Declare comparison types
+    GT = ">"
+    LT = "<"
+    EQ = "="
+
+    def __init__(self, filename) -> None:
+        pass
+
+    def put_artifacts(self, artifacts, kwargs) -> None:
+        pass
+
+    def get_artifacts(self, query):
+        pass
+
+    def inspect_artifacts(self):
+        pass

dsi/backends/gufi.py

@@ -0,0 +1,92 @@
+import subprocess
+
+
+# Holds table name and data properties
+from dsi.backends.filesystem import Filesystem
+
+
+class DataType:
+    name = "DEFAULT"
+    properties = {}
+    units = {}
+
+
+class Gufi(Filesystem):
+    '''
+    GUFI Datastore
+    '''
+
+    prefix = ""
+    index = ""
+    dbfile = ""
+    table = ""
+    column = ""
+    isVerbose = False
+
+    """
+    prefix: prefix to GUFI commands
+    index: directory with GUFI indexes
+    dbfile: sqlite db file from DSI
+    table: table name from the DSI db we want to join on
+    column: column name from the DSI db to join on
+    """
+
+    def __init__(self, prefix, index, dbfile, table, column, verbose=False):
+        '''
+        prefix: prefix to GUFI commands
+        index: directory with GUFI indexes
+        dbfile: sqlite db file from DSI
+        table: table name from the DSI db we want to join on
+        column: column name from the DSI db to join on
+        verbose: print debugging statements or not
+        '''
+
+        super().__init__(dbfile)
+        # prefix is the prefix to the GUFI installation
+        self.prefix = prefix
+        # index is the directory where the GUFI indexes are stored
+        self.index = index
+        # dbfile is the dsi database file that we wish to use
+        self.dbfile = dbfile
+        # table is the dsi database table name that we wish to use
+        self.table = table
+        # column is the dsi column name for a file's inode
+        self.column = column
+
+        self.isVerbose = verbose
+
+    # Query GUFI and DSI db
+    def get_artifacts(self, query):
+        '''
+        Retrieves GUFI's metadata joined with a dsi database
+        query: an sql query into the dsi_entries table
+        '''
+
+        resout = self._run_gufi_query(query)
+        if self.isVerbose:
+            print(resout)
+
+        return resout
+
+    def put_artifacts(self, query):
+        pass
+
+    # Runs the gufi query command
+    def _run_gufi_query(self, sqlstring):
+        '''
+        Calls the qufy_query command to run the sql query
+        sqlstring: the query into the dsi_entries table
+        '''
+
+        # Create the string for the -Q option that specifies the dsi db file,
+        # the dsi db table name, and the dsi db inode column name.
+        _ = self.dbfile + " " + self.table + " " + self.column + " inode"
+
+        # Run the GUFI query command
+        result = subprocess.run([self.prefix + "/gufi_query", "-d", "|", "-Q", self.dbfile,
+                                self.table, self.column, "inode", "-E", sqlstring, self.index],
+                                capture_output=True, text=True)
+        return result.stdout
+
+    def close(self):
+        pass

dsi/backends/parquet.py

@@ -0,0 +1,101 @@
+import pyarrow as pa
+from pyarrow import parquet as pq
+import subprocess
+
+from dsi.backends.filesystem import Filesystem
+
+
+class Parquet(Filesystem):
+    """
+    Support for a Parquet back-end.
+
+    Parquet is a convenient format when metadata are larger than SQLite supports.
+    """
+
+    def __init__(self, filename, **kwargs):
+        super().__init__(filename=filename)
+        self.filename = filename
+        try:
+            self.compression = kwargs['compression']
+        except KeyError:
+            self.compression = None
+
+    def get_artifacts(self):
+        """Get Parquet data from filename."""
+        table = pq.read_table(self.filename)
+        resout = table.to_pydict()
+        return resout
+
+    def put_artifacts(self, collection):
+        """Put artifacts into file at filename path."""
+        table = pa.table(collection)
+        pq.write_table(table, self.filename, compression=self.compression)
+
+    @staticmethod
+    def get_cmd_output(cmd: list) -> str:
+        """
+        Runs a given command and returns the stdout if successful.
+
+        If stderr is not empty, an exception is raised with the stderr text.
+        """
+        proc = subprocess.run(cmd, capture_output=True, shell=True)
+        if proc.stderr != b"":
+            raise Exception(proc.stderr)
+        return proc.stdout.strip().decode("utf-8")
+
+    def inspect_artifacts(self, collection, interactive=False):
+        import nbconvert as nbc
+        import nbformat as nbf
+
+        """Populate a Jupyter notebook with tools required to look at Parquet data."""
+        nb = nbf.v4.new_notebook()
+        text = """\
+        # This notebook was auto-generated by a DSI Backend for Parquet.
+        # Execute the Jupyter notebook cells below and interact with "df"
+        # to explore your data.
+        """
+        code1 = """\
+        import pandas as pd
+        df = pd.read_parquet('{}')
+        df.head()
+        """.format(self.filename)
+
+        code2 = """\
+        df.info()
+        """
+
+        code3 = """\
+        df.describe()
+        """
+
+        nb['cells'] = [nbf.v4.new_markdown_cell(text),
+                       nbf.v4.new_code_cell(code1),
+                       nbf.v4.new_code_cell(code2),
+                       nbf.v4.new_code_cell(code3)]
+
+        fname = 'dsi_parquet_backend_output.ipynb'
+
+        print('Writing Jupyter notebook...')
+        with open(fname, 'w') as fh:
+            nbf.write(nb, fh)
+
+        # open the jupyter notebook for static page generation
+        with open(fname, 'r', encoding='utf-8') as fh:
+            nb_content = nbf.read(fh, as_version=4)
+        # Init executor for notebook
+        run_nb = nbc.preprocessors.ExecutePreprocessor(timeout=-1) # No timeout
+        # Execute the notebook
+        run_nb.preprocess(nb_content, {'metadata':{'path':'.'}})
+
+        if interactive:
+            print('Opening Jupyter notebook...')
+            self.get_cmd_output(cmd=['jupyter-lab ./dsi_parquet_backend_output.ipynb'])
+        else:
+#            self.get_cmd_output(cmd=['jupyter nbconvert --to html {}'.format(fname)])
+            # Init HTML exporter
+            html_exporter = nbc.HTMLExporter()
+            html_content,_ = html_exporter.from_notebook_node(nb_content)
+            # Save HTML file
+            html_filename = 'dsi_parquet_backend_output.html'
+            with open(html_filename, 'w', encoding='utf-8') as fh:
+                fh.write(html_content)

dsi/backends/sqlalchemy.py

@@ -0,0 +1,43 @@
+from typing import List
+from typing import Optional
+from sqlalchemy import ForeignKey
+from sqlalchemy import String
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+from sqlalchemy.orm import relationship
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+import csv
+import json
+import re
+import yaml
+import toml
+
+from dsi.backends.filesystem import Filesystem
+
+class SqlAlchemy(Filesystem):
+    filename = "sqlite:///fs.db"
+    engine = None
+
+    def __init__(self, filename, base):
+        self.filename = filename
+        self.engine = create_engine(filename, echo=True)
+        base.metadata.create_all(self.engine)
+
+    def put_artifacts(self, artifact_list):
+        with Session(self.engine) as session:
+            session.add_all(artifact_list)
+            session.commit()
+
+    def query(self, stmt):
+        results = []
+        with Session(self.engine) as session:
+            for obj in session.scalars(stmt):
+                results.append(obj)
+
+        return results
+    
+    def close(self):
+        if self.engine:
+            self.engine.dispose()