dsff 1.0.7__py3-none-any.whl → 1.2.0__py3-none-any.whl

dsff/formats/__init__.py

@@ -0,0 +1,346 @@
+# -*- coding: UTF-8 -*-
+import builtins
+import logging
+import openpyxl.reader.excel as excelr
+import types
+from ast import literal_eval
+from datetime import datetime
+from functools import wraps
+from getpass import getuser
+from openpyxl import load_workbook, Workbook
+from openpyxl.styles import Alignment, Font
+from openpyxl.utils import get_column_letter
+from zipfile import BadZipFile, ZipFile
+
+from .__common__ import *
+from .arff import *
+from .csv import *
+from .dataset import *
+from .db import *
+if PYARROW:  # pragma: no cover
+    from .pa import *
+
+
+__all__ = ["DSFF"]
+
+_FORMAT_TEXT_ALIAS = {'arff': "ARFF", 'csv': "CSV", 'db': "SQL", 'orc': "ORC"}
+_NO_EXT_FORMAT     = ["dataset"]
+
+
+for name, etype in [("BadDsffFile", "OSError"), ("BadInputData", "ValueError"), ("EmptyDsffFile", "ValueError")]:
+    if not hasattr(builtins, name):
+        exec(f"class {name}({etype}): __module__ = 'builtins'")
+        setattr(builtins, name, locals()[name])
+
+
+def _bind_from(dsff):
+    def _adapt_name(f):
+        def _wrapper(*a, **kw):
+            path = None
+            if len(a) > 0:
+                path, a = a[0], a[1:]
+            elif 'path' in kw:
+                path = kw.pop('path')
+            path = _fix_path(path or dsff.name, f.__name__.split("_", 1)[1])
+            if path == INMEMORY:
+                raise ValueError("no path specified")
+            dsff.logger.debug(f"creating DSFF from {path}...")
+            return f(dsff, path, *a, **kw)
+        setattr(dsff, f.__name__, _wrapper)
+        return _wrapper
+    return _adapt_name
+
+
+def _bind_to(dsff):
+    def _is_empty(f):
+        def _wrapper(*a, **kw):
+            if len(dsff) == 0:
+                raise EmptyDsffFile("No data")
+            path = None
+            if len(a) > 0:
+                path, a = a[0], a[1:]
+            elif 'path' in kw:
+                path = kw.pop('path')
+            path = _fix_path(path or ("undefined" if dsff.name == INMEMORY else dsff.name),
+                             fmt := f.__name__.split("_", 1)[1])
+            dsff.logger.debug(f"converting DSFF to {path}..." if 'text' not in kw or kw['text'] else \
+                              f"converting DSFF to {_FORMAT_TEXT_ALIAS.get(fmt, fmt.capitalize())}...")
+            return f(dsff, path, *a, **kw)
+        setattr(dsff, f.__name__, _wrapper)
+        return _wrapper
+    return _is_empty
+
+
+def _fix_path(path, ext):
+    from os.path import expanduser, splitext
+    if path == INMEMORY:
+        return path
+    path = splitext(expanduser(path))[0]
+    if ext is not None and ext not in _NO_EXT_FORMAT and not path.endswith(f".{ext}"):
+        path += f".{ext}"
+    return path
+
+
+class DSFF:
+    """ DataSet File Format.
+    
+    Modes:          r   r+  w   w+
+        --------------------------
+        read        *   *       *
+        write           *   *   *
+        create              *   *
+        truncate            *   *
+    
+    Important note: XSLX format has a limitation of 1,048,576 rows per sheet.
+    """
+    def __init__(self, path=None, mode=None, logger=None):
+        if mode is None:
+            mode = "rw"[path in [None, INMEMORY]]
+        if re.match(r"[rw]\+?$", mode) is None:
+            raise ValueError("Mode should be one of: r, r+, w, w+")
+        self.__change = False
+        self.__logger = logger or logging.getLogger("DSFF")
+        self.__name = None
+        self.__path = path
+        self.__mode = mode
+        # depending on the mode, bind the necessary methods
+        if mode in ["r+", "w", "w+"]:
+            self.save = types.MethodType(lambda dsff: dsff._DSFF__save(), self)
+        self.logger.debug("binding write methods")
+        for name, obj in globals().items():
+            if name.startswith("from_"):
+                _bind_from(self)(obj)
+        self.logger.debug("binding read methods")
+        for name, obj in globals().items():
+            if name.startswith("to_"):
+                _bind_to(self)(obj)
+        # perform checks
+        if mode in ["r", "r+"]:
+            if path is None:
+                raise ValueError("No input path to a .dsff file provided")
+            if path != INMEMORY and not isfile(path):
+                raise FileNotFoundError("Input .dsff does not exist")            
+        # if the target path exists and is a file, open it
+        if mode in ["r", "r+"] and path != INMEMORY:
+            # disable archive validation as it does not recognize '.dsff'
+            tmp = excelr._validate_archive
+            excelr._validate_archive = lambda f: ZipFile(f, 'r')
+            try:
+                self.__wb = load_workbook(path)
+            except BadZipFile:
+                raise BadDsffFile("File is not a DSFF file")
+            finally:
+                excelr._validate_archive = tmp
+            # check that the file has only 2 worksheets: 'data' and 'features'
+            if [ws._WorkbookChild__title for ws in self.__wb.worksheets] != ["data", "features"]:
+                raise BadDsffFile("File is not a DSFF file")
+            # check that the 'features' worksheet has 2 columns: 'name' and 'description'
+            for headers in self.__wb['features'].rows:
+                if len(headers) != 2 or headers[0].value != "name" or headers[1].value != "description":
+                    raise BadDsffFile("The features worksheet does not comply with DSFF")
+                break
+            return
+        # otherwise, create a new workbook with the default worksheets
+        if isfile(self.path):
+            remove(self.path)  # re-create
+        self.__wb = Workbook()
+        del self.__wb['Sheet']  # remove the default sheet
+        for ws in ["data", "features"]:
+            self.__wb.create_sheet(ws)
+    
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, *args):
+        self.__save()
+        self.close()
+    
+    def __getitem__(self, name):
+        if name in ["data", "features"]:
+            return self.__wb[name]
+        # the common property 'description' is used to store the metadata of the dataset, hence 'description' can be
+        #  used as a key in the metadata but not from the common properties
+        try:
+            if name != "description":
+                return getattr(self.__wb.properties, name)
+        except AttributeError:
+            pass
+        return self.metadata[name]
+    
+    def __len__(self):
+        return len(self.data)
+    
+    def __setitem__(self, name, value):
+        if name in ["data", "features"]:
+            raise ValueError(f"'{name}' is a name reserved for a worksheet")
+        # see the note from __getitem__ related to 'description'
+        if hasattr(self.__wb.properties, name) and name != "description":
+            setattr(self.__wb.properties, name, value)
+        d = self.metadata
+        d[name] = value
+        self.__wb.properties.description = json.dumps(d)
+        self.__change = True
+    
+    def __eval(self, v):
+        try:
+            return literal_eval(v)
+        except (SyntaxError, ValueError):
+            return v
+    
+    def __save(self):
+        if self.mode == "r" or self.path == INMEMORY:
+            return
+        if self.__change:
+            props = self.__wb.properties
+            if props.creator is None or props.creator == "openpyxl":
+                props.creator = getuser()
+            props.title = self.name
+            props.description = self.metadata
+            if isfile(self.path) and self.mode.startswith("w"):
+                remove(self.path)
+            self.__wb.save(self.path)
+            self.__change = False
+    
+    def _to_table(self):
+        if not PYARROW:
+            raise NotImplementedError("This method is available only wheh Pyarrow is installed")
+        headers = self.data[0]
+        try:
+            l = headers.index("label")
+        except ValueError:
+            l = -1
+        cols = [pyarrow.array(col if i != l else [[c, None][c == MISSING_TOKEN] for c in col]) \
+                for i, col in enumerate(zip(*self.data[1:]))]
+        meta = {k: v for k, v in self.features.items()}
+        meta['__metadata__'] = str(self.metadata).encode()
+        schema_with_meta = pyarrow.schema(pyarrow.Table.from_arrays(cols, names=headers).schema, metadata=meta)
+        return pyarrow.Table.from_arrays(cols, schema=schema_with_meta)
+    
+    def close(self):
+        self.__wb.close()
+    
+    def write(self, data=None, features=None, metadata=None, missing="?"):
+        """ Write data and/or features and/or metadata to the workbook.
+        
+        :param data:     matrix of data (including headers) OR path to data.csv OR path to Dataset folder
+        :param features: dictionary of features' names and descriptions OR path to features.json
+        :param metadata: dictionary of dataset's metadata OR path to metadata.json
+        """
+        # get the cell coordinate from (X,Y) coordinates (e.g. (1,2) => "B1")
+        coord = lambda x, y: ws.cell(x+1, y+1).coordinate
+        # private function to auto-adjust column widths
+        def autoadjust(ws):
+            col_widths = []
+            for row in ws.rows:
+                if len(col_widths) == 0:
+                    col_widths = len(row) * [0]
+                for i, cell in enumerate(row):
+                    col_widths[i] = max(col_widths[i], len(str(cell.value)))
+            for i, w in enumerate(col_widths):
+                ws.column_dimensions[get_column_letter(i+1)].width = w
+        # if the first argument is a folder, assume it is a specific folder structure as follows:
+        #   [name]
+        #     +-- data.csv            # metadata and labels of the executable
+        #     +-- features.json       # dictionary of selected features and their descriptions
+        #     +-- metadata.json       # simple statistics about the dataset
+        if data is not None and not isinstance(data, (list, dict)) and isdir(expanduser(data)):
+            self.__path, d = self.__path or basename(data), expanduser(data)
+            data, features, metadata = join(d, "data.csv"), join(d, "features.json"), join(d, "metadata.json")
+        # handle data first
+        if data is not None:
+            self.__logger.debug("writing data to DSFF...")
+            ws, d = self.__wb['data'], data
+            if not isinstance(d, list):
+                if isfile(expanduser(d)) and splitext(d)[1] == ".csv":
+                    with open(expanduser(d)) as f:
+                        d = [r for r in csvmod.reader(f, delimiter=CSV_DELIMITER)]
+                else:
+                    raise BadInputData("'data' is not a list")
+            for r, row in enumerate(d):
+                for c, value in enumerate(row):
+                    c = coord(r, c)
+                    ws[c] = str({None: missing}.get(value, value))
+                    if r == 0:
+                        ws[c].alignment = Alignment(horizontal="center")
+                        ws[c].font = Font(bold=True)
+            autoadjust(ws)
+            self.__change = True
+        # then handle features dictionary
+        if len(features or {}) > 0:
+            self.__logger.debug("writing features to DSFF...")
+            ws, headers, d = self.__wb['features'], ["name", "description"], features
+            if not isinstance(d, dict):
+                if isfile(expanduser(d)) and basename(d) == "features.json":
+                    with open(expanduser(d)) as f:
+                        d = json.load(f)
+                else:
+                    raise BadInputData("'features' is not a dictionary")
+            try:
+                for c, header in enumerate(headers):
+                    c = coord(0, c)
+                    ws[c] = header
+                    ws[c].alignment = Alignment(horizontal="center")
+                    ws[c].font = Font(bold=True)
+                for r, pair in enumerate(d.items()):
+                    ws[coord(r+1, 0)] = pair[0]
+                    ws[coord(r+1, 1)] = pair[1]
+                autoadjust(ws)
+                self.__change = True
+            except Exception as e:
+                raise BadInputData(f"Unexpected error while parsing 'features' ({e})")
+        # finally handle metadata dictionary
+        if len(metadata or {}) > 0:
+            self.__logger.debug("writing metadata to DSFF...")
+            d = metadata
+            if not isinstance(d, dict):
+                if isfile(expanduser(d)) and basename(d) == "metadata.json":
+                    with open(expanduser(d)) as f:
+                        d = json.load(f)
+                else:
+                    raise BadInputData("'metadata' is not a dictionary")
+            try:
+                self.__wb.properties.description = json.dumps(d)
+                self.__change = True
+            except Exception as e:
+                raise BadInputData(f"Unexpected error while parsing 'metadata' ({e})")
+        self.__save()
+    
+    @property
+    def data(self):
+        return [[self.__eval(c.value) for c in cells] for cells in self.__wb['data'].rows]
+    
+    @property
+    def features(self):
+        return {cells[0].value: cells[1].value for i, cells in enumerate(self.__wb['features'].rows) if i > 0}
+    
+    @property
+    def headers(self):
+        return list(self.features.keys())
+    
+    @property
+    def logger(self):
+        return self.__logger
+    
+    @property
+    def metadata(self):
+        return json.loads((self.__wb.properties.description or "{}").replace("'", "\""))
+    
+    @property
+    def mode(self):
+        return self.__mode
+    
+    @property
+    def name(self):
+        return self.__name or self.__wb.properties.title or splitext(basename(self.path))[0]
+    
+    @name.setter
+    def name(self, name):
+        self.__name = name
+    
+    @property
+    def path(self):
+        p = self.__path or "undefined"
+        if p != INMEMORY and not p.endswith(".dsff"):
+            p += ".dsff"
+        return p
+

dsff/{arff.py → formats/arff.py}

@@ -7,18 +7,12 @@ __all__ = ["from_arff", "to_arff"]
 
 def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
     """ Populate the DSFF file from an ARFF file. """
-    path = expanduser(path or dsff.name)
-    if not path.endswith(".arff"):
-        path += ".arff"
-    dsff.logger.debug("creating DSFF from ARFF file...")
-    d = []
+    d, features = [], {}
     with open(path) as f:
         relation, attributes, data = False, [False, False], False
         for n, l in enumerate(f, 1):
-            l = l.strip()
-            # ignore comments before @RELATION
-            if l.startswith("#"):
-                continue
+            l, pf = l.strip(), f"Line {n}: "
+            # the file shall start with "@RELATION"
             if not relation:
                 if l.startswith("@RELATION "):
                     relation = True
@@ -26,9 +20,17 @@ def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
                         dsff['title'] = re.match(r"@RELATION\s+('[^']*'|\"[^\"]*\")$", l).group(1).strip("'\"")
                         continue
                     except Exception as e:
-                        raise BadInputData("Line %d: failed on @RELATION (%s)" % (n, e))
+                        raise BadInputData(f"{pf}failed on @RELATION ({e})")
                 else:
-                    raise BadInputData("Line %d: did not find @RELATION" % n)
+                    raise BadInputData(f"{pf}did not find @RELATION")
+            # get metadata and feature descriptions from comments
+            if l.startswith("%"):
+                if re.match(r"^\%\s+metadata\s*\:\s*\{.*\}$", l):
+                    dsff.write(metadata=literal_eval(l.split(":", 1)[1]))
+                elif (m := re.match(r"^\%\s+(.*?)\s*\:\s*(.*?)$", l)):
+                    name, descr = m.groups()
+                    features[name] = descr
+                continue
             # then ignore blank lines
             if l == "":
                 if attributes[0] and not attributes[1]:
@@ -43,7 +45,7 @@ def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
                     # start the attributes block
                     d.append([])
                 if attributes[1]:
-                    raise BadInputData("Line %d: found @ATTRIBUTE out of the attributes block)" % n)
+                    raise BadInputData(f"{pf}found @ATTRIBUTE out of the attributes block)")
                 try:
                     header = re.match(r"@ATTRIBUTE\s+([^\s]+)\s+[A-Z]+$", l).group(1)
                     if header == "class":
@@ -51,16 +53,16 @@ def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
                     d[0].append(header)
                     continue
                 except AttributeError:
-                    raise BadInputData("Line %d: failed on @ATTRIBUTE (bad type)" % n)
+                    raise BadInputData(f"{pf}failed on @ATTRIBUTE (bad type)")
             if not data:
                 if l == "@DATA":
                     data = True
                     continue
                 else:
-                    raise BadInputData("Line %d: did not find @DATA where expected" % n)
+                    raise BadInputData(f"{pf}did not find @DATA where expected")
             row = list(map(lambda x: x.strip("'\""), re.split(r",\s+", l)))
             if len(row) != n_cols:
-                raise BadInputData("Line %d: this row does not match the number of columns" % n)
+                raise BadInputData(f"{pf}this row does not match the number of columns")
             d.append(row)
     for i in range(n_cols):
         values = []
@@ -75,23 +77,14 @@ def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
             for j, row in enumerate(d):
                 if j > 0:
                     row[i] = {'0': "False", '1': "True"}[row[i]]
-    dsff.write(d)
-    features = {}
-    for headers in dsff['data'].rows:
-        for header in headers:
-            features[header.value] = ""
-        break
+    dsff.write(data=d)
     dsff.write(features=features)
 
 
 def to_arff(dsff, path=None, target=TARGET_NAME, exclude=DEFAULT_EXCL, missing=MISSING_TOKEN, text=False):
     """ Output the dataset in ARFF format, suitable for use with the Weka framework, saved as a file or output as a
          string. """
-    path = splitext(expanduser(path or dsff.name))[0]
-    if not path.endswith(".arff"):
-        path += ".arff"
     name = splitext(basename(path))[0]
-    dsff.logger.debug("extracting data from DSFF to ARFF file...")
     _d = lambda c: {None: missing, '': "-", 'False': "0", 'True': "1"}.get(c.value, c.value)
     _sanitize_n = lambda c: _d(c).replace("<", "[lt]").replace(">", "[gt]")
     _sanitize_v = lambda c: _d(c)[1:].strip("\"'") if _d(c).startswith("=") else _d(c)
@@ -119,37 +112,40 @@ def to_arff(dsff, path=None, target=TARGET_NAME, exclude=DEFAULT_EXCL, missing=M
     types = []
     # parse labels, data types and relevant data
     for i, row in enumerate(data.rows):
-        if i > 0:
-            if len(types) == 0:
-                labels = ["0", "1"] if i_target == -1 else \
-                         list(set(_sanitize_v(row[i_target]) for k, row in enumerate(data) if k > 0))
-                labels = [x for x in labels if x != missing]
-                # compute types
-                for j, cell in enumerate(row):
-                    v = _sanitize_v(cell)
-                    try:
-                        float(v)
-                        t = "NUMERIC"
-                    except ValueError:
-                        t = "STRING"
-                    types.append(t)
-                # filter data types based on the relevant columns
-                types = [t for k, t in enumerate(types) if k != i_target and k not in h_excl] + \
-                        [types[i_target] if i_target > -1 else []]
-                # compute the list of ARFF attribute lines based on the column names and data types
-                a = [("@ATTRIBUTE {: <%s} {}" % mlen_h).format("class" if j == len(types)-1 else \
-                      _sanitize_n(headers[j]), t) for j, t in enumerate(types)]
-                mlen_c = [0] * len(types)
-            # filter data based on the relevant columns
-            row = [_sanitize_v(x) for k, x in enumerate(row) if k != i_target and k not in h_excl] + \
-                  ([_sanitize_v(row[i_target])] if i_target > -1 else [])
-            # compute the maximum length for each column
-            mlen_c = [max(x, len(row[k]) if types[k] == "NUMERIC" else len(row[k])+2) for k, x in enumerate(mlen_c)]
-            d.append(row)
+        if i == 0:
+            continue  # do not process headers
+        if len(types) == 0:
+            labels = ["0", "1"] if i_target == -1 else \
+                     list(set(_sanitize_v(row[i_target]) for k, row in enumerate(data) if k > 0))
+            labels = [x for x in labels if x != missing]
+            # compute types
+            for j, cell in enumerate(row):
+                v = _sanitize_v(cell)
+                try:
+                    float(v)
+                    t = "NUMERIC"
+                except ValueError:
+                    t = "STRING"
+                types.append(t)
+            # filter data types based on the relevant columns
+            types = [t for k, t in enumerate(types) if k != i_target and k not in h_excl] + \
+                    [types[i_target] if i_target > -1 else []]
+            # compute the list of ARFF attribute lines based on the column names and data types
+            a = [("@ATTRIBUTE {: <%s} {}" % mlen_h).format("class" if j == len(types)-1 else \
+                  _sanitize_n(headers[j]), t) for j, t in enumerate(types)]
+            mlen_c = [0] * len(types)
+        # filter data based on the relevant columns
+        row = [_sanitize_v(x) for k, x in enumerate(row) if k != i_target and k not in h_excl] + \
+              ([_sanitize_v(row[i_target])] if i_target > -1 else [])
+        # compute the maximum length for each column
+        mlen_c = [max(x, len(row[k]) if types[k] == "NUMERIC" else len(row[k])+2) for k, x in enumerate(mlen_c)]
+        d.append(row)
     # format the resulting data and output the ARFF
-    d = "\n".join(" ".join(("{: <%s}" % (mlen_c[k]+1)).format((x if types[k] == "NUMERIC" or x == MISSING_TOKEN else \
-                            "'%s'" % x) + ",") for k, x in enumerate(row)).rstrip(" ,") for row in d)
-    arff = "@RELATION \"{}\"\n\n{}\n\n@DATA\n{}".format(name, "\n".join(a), d)
+    d = (nl := "\n").join(" ".join(("{: <%s}" % (mlen_c[k]+1)).format((x if types[k] == "NUMERIC" or \
+                       x == MISSING_TOKEN else "'%s'" % x) + ",") for k, x in enumerate(row)).rstrip(" ,") for row in d)
+    arff = f"@RELATION \"{name}\"\n\n{nl.join(a)}\n\n@DATA\n{d}\n\n" \
+           f"{['', f'% metadata: {json.dumps(dsff.metadata)}'][len(dsff.metadata) > 0]}\n\n" \
+           f"{nl.join(f'% {name}: {descr}' for name, descr in dsff.features.items())}"
     if text:
         return arff
     with open(path, 'w+') as f:

dsff/{csv.py → formats/csv.py}

@@ -6,15 +6,13 @@ __all__ = ["from_csv", "to_csv"]
 
 
 def from_csv(dsff, path=None, exclude=DEFAULT_EXCL):
-    """ Populate the DSFF file from an ARFF file. """
-    path = expanduser(path or dsff.name)
-    if not path.endswith(".csv"):
-        path += ".csv"
-    dsff.logger.debug("creating DSFF from CSV file...")
+    """ Populate the DSFF file from a CSV file. """
     dsff.write(path)
     features = {}
     for headers in dsff['data'].rows:
         for header in headers:
+            if header.value in exclude:
+                continue
             features[header.value] = ""
         break
     dsff.write(features=features)
@@ -22,14 +20,10 @@ def from_csv(dsff, path=None, exclude=DEFAULT_EXCL):
 
 def to_csv(dsff, path=None, text=False):
     """ Create a CSV from the data worksheet, saved as a file or output as a string. """
-    path = splitext(expanduser(path or dsff.name))[0]
-    if not path.endswith(".csv"):
-        path += ".csv"
-    dsff.logger.debug("extracting data from DSFF to CSV file...")
     with (StringIO() if text else open(path, 'w+')) as f:
         writer = csvmod.writer(f, delimiter=";")
-        for cells in dsff['data'].rows:
-            writer.writerow([dsff._DSFF__eval(c.value) for c in cells])
+        for row in dsff.data:
+            writer.writerow(row)
         if text:
             return f.getvalue()
 

dsff/{dataset.py → formats/dataset.py}

@@ -6,31 +6,23 @@ __all__ = ["from_dataset", "to_dataset"]
 
 
 def from_dataset(dsff, path=None):
-    """ Populate the DSFF file from an ARFF file. """
-    path = expanduser(path or dsff.name)
-    dsff.logger.debug("creating DSFF from (Fileless)Dataset folder...")
+    """ Populate the DSFF file from a Dataset structure. """
     if not isdir(path):
         raise BadInputData("Not a folder")
     else:
-        missing = []
-        for f in ["data.csv", "features.json", "metadata.json"]:
-            if not isfile(join(path, f)):
-                missing.append(f)
-        if len(missing) > 0:
-            raise BadInputData("Not a valid dataset folder (missing: %s)" % ", ".join(missing))
+        if len(missing := [f for f in ["data.csv", "features.json", "metadata.json"] if not isfile(join(path, f))]) > 0:
+            raise BadInputData(f"Not a valid dataset folder (missing: {', '.join(missing)})")
     dsff.write(path)
 
 
 def to_dataset(dsff, path=None):
-    """ Create a dataset folder according to the Dataset structure ;
+    """ Create a dataset folder according to the following structure ;
     name
      +-- data.csv
      +-- features.json
      +-- metadata.json
     """
-    path = splitext(expanduser(path or dsff.name))[0]
     makedirs(path, exist_ok=True)
-    dsff.logger.debug("converting DSFF to (Fileless)Dataset folder...")
     # handle data first
     dsff.logger.debug("> making data.csv...")
     with open(join(path, "data.csv"), 'w+') as f:

dsff/formats/db.py

@@ -0,0 +1,76 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import *
+
+
+__all__ = ["from_db", "to_db"]
+
+
+def from_db(dsff, path=None, exclude=DEFAULT_EXCL):
+    """ Populate the DSFF file from a SQLDB file. """
+    from json import loads
+    from sqlite3 import connect
+    conn = connect(path)
+    cursor = conn.cursor()
+    # list tables
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = [table[0] for table in cursor.fetchall()]
+    if not all(t in tables for t in ["data", "features", "metadata"]):  # pragma: no cover
+        raise BadInputData("The target SQLDB does not have the right format")
+    # import data
+    cursor.execute("PRAGMA table_info('data')")
+    headers = [[col[1] for col in cursor.fetchall()]]
+    cursor.execute("SELECT * FROM data;")
+    dsff.write(headers + [r for r in cursor.fetchall()])
+    # import feature definitions
+    cursor.execute("SELECT name,description FROM features;")
+    dsff.write(features={r[0]: r[1] for r in cursor.fetchall()})
+    # import metadata
+    cursor.execute("SELECT key,value FROM metadata;")
+    dsff.write(metadata={r[0]: loads(r[1]) if isinstance(r[1], str) else r[1] for r in cursor.fetchall()})
+    conn.close()
+
+
+def to_db(dsff, path=None, text=False, primary_index=0):
+    """ Create a SQLDB from the data worksheet, saved as a file or output as a string. """
+    from json import dumps
+    from sqlite3 import connect
+    fields = []
+    rows = (data := dsff['data']).rows
+    headers, first = [c.value for c in next(rows)], next(rows)
+    for i, pair in enumerate(zip(headers, first)):
+        header, cell = pair
+        try:
+            dtype = {int: "INTEGER", float: "REAL", bool: "INTEGER"}[type(dsff._DSFF__eval(cell.value))]
+        except (KeyError, ValueError):
+            dtype = "TEXT"
+        fields.append(f"{header} {dtype}{['',' PRIMARY KEY'][i==primary_index]}")
+    # create the database
+    conn = connect(":memory:" if text else path)
+    cursor = conn.cursor()
+    # create and populate the data table
+    cursor.execute("CREATE TABLE IF NOT EXISTS data ({fields});" \
+                   .format(fields="\n    ".join(f"{f}," for f in fields).rstrip(",")))
+    cursor.executemany("INSERT INTO data ({fields}) VALUES ({tokens});"\
+                       .format(fields=",".join(headers), tokens=",".join(["?"]*len(headers))),
+                       [[v.value for v in row] for i, row in enumerate(data.rows) if i > 0])
+    # create and populate the features table
+    cursor.execute("CREATE TABLE IF NOT EXISTS features (name TEXT PRIMARY KEY, description TEXT);")
+    cursor.executemany("INSERT INTO features (name, description) VALUES (?, ?);",
+                       [(r[0].value, r[1].value) for i, r in enumerate(dsff['features'].rows) if i > 0])
+    # create and populate the metadata table
+    cursor.execute("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value JSON);")
+    cursor.executemany("INSERT INTO metadata (key, value) VALUES (?, ?);",
+                       [(k, dumps(v)) for k, v in dsff.metadata.items()])
+    conn.commit()
+    if text:
+        sql = {}
+        # extract SQL code
+        cursor.execute(f"SELECT sql FROM sqlite_master WHERE type='table';")
+        sql['table'] = "\n".join(row[0] for row in cursor.fetchall())
+        for t in ["data", "features", "metadata"]:
+            cursor.execute(f"SELECT * FROM {t};")
+            sql[t] = "\n".join(f"INSERT INTO {t} VALUES ('{row[0]}', '{row[1]}');" for row in cursor.fetchall())
+        # combine all SQL
+        return "\n".join(sql.values())
+    conn.close()
+