dsff 1.2.0__tar.gz → 1.2.3__tar.gz

{dsff-1.2.0/src/dsff.egg-info → dsff-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsff
-Version: 1.2.0
+Version: 1.2.3
 Summary: DataSet File Format (DSFF)
 Author-email: Alexandre D'Hondt <alexandre.dhondt@gmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -696,8 +696,10 @@ Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: openpyxl
-Provides-Extra: extra
-Requires-Dist: pyarrow; extra == "extra"
+Provides-Extra: all
+Requires-Dist: dsff[arrow]; extra == "all"
+Provides-Extra: arrow
+Requires-Dist: pyarrow; extra == "arrow"
 Dynamic: license-file
 
 <p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/imgs/logo.png"></p>
@@ -738,9 +740,34 @@ pip install --user dsff
 ```python
 >>> import dsff
 >>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
-    f.to_dataset()  # creates ./[dsff-title] with data.csv, features.json and metadata.json
+    f.to_dataset()  # creates ./my-dataset with data.csv, features.json and metadata.json
 ```
 
+**
+
+## Extensions
+
+**Install all available extensions**
+
+```sh
+pip install --user dsff[all]
+```
+
+**Dealing with [Apache Arrow](https://arrow.apache.org/) formats**
+
+```sh
+pip install --user dsff[arrow]
+```
+
+```python
+>>> import dsff
+>>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
+    f.to_feather()  # creates ./my-dataset.feather
+    f.to_orc()      # creates ./my-dataset.orc
+    f.to_parquet()  # creates ./my-dataset.parquet
+```
+
+
 ## Related Projects
 
 You may also like these:

{dsff-1.2.0 → dsff-1.2.3}/README.md

@@ -36,9 +36,34 @@ pip install --user dsff
 ```python
 >>> import dsff
 >>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
-    f.to_dataset()  # creates ./[dsff-title] with data.csv, features.json and metadata.json
+    f.to_dataset()  # creates ./my-dataset with data.csv, features.json and metadata.json
 ```
 
+**
+
+## Extensions
+
+**Install all available extensions**
+
+```sh
+pip install --user dsff[all]
+```
+
+**Dealing with [Apache Arrow](https://arrow.apache.org/) formats**
+
+```sh
+pip install --user dsff[arrow]
+```
+
+```python
+>>> import dsff
+>>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
+    f.to_feather()  # creates ./my-dataset.feather
+    f.to_orc()      # creates ./my-dataset.orc
+    f.to_parquet()  # creates ./my-dataset.parquet
+```
+
+
 ## Related Projects
 
 You may also like these:

dsff-1.2.3/docs/coverage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="114" height="20" role="img" aria-label="coverage: 93.24%"><title>coverage: 93.24%</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="114" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="61" height="20" fill="#555"/><rect x="61" width="53" height="20" fill="#4c1"/><rect width="114" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="315" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="510">coverage</text><text x="315" y="140" transform="scale(.1)" fill="#fff" textLength="510">coverage</text><text aria-hidden="true" x="865" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">93.24%</text><text x="865" y="140" transform="scale(.1)" fill="#fff" textLength="430">93.24%</text></g></svg>

{dsff-1.2.0 → dsff-1.2.3}/pyproject.toml

@@ -34,9 +34,8 @@ dependencies = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
-extra = [
-  "pyarrow",
-]
+all = ["dsff[arrow]"]
+arrow = ["pyarrow"]
 
 [project.readme]
 file = "README.md"

dsff-1.2.3/src/dsff/VERSION.txt

@@ -0,0 +1 @@
+1.2.3

dsff-1.2.3/src/dsff/formats/__common__.py

@@ -0,0 +1,56 @@
+# -*- coding: UTF-8 -*-
+import csv as csvmod
+import json
+import re
+from ast import literal_eval
+from io import BytesIO, StringIO
+from os import makedirs, remove
+from os.path import basename, exists, expanduser, isfile, isdir, join, splitext
+try:  # pragma: no cover
+    import pyarrow
+    import pyarrow.feather as feather
+    import pyarrow.orc as orc
+    import pyarrow.parquet as parquet
+    import pandas
+    PYARROW = True
+except ImportError:  # pragma: no cover
+    PYARROW = False
+
+
+__all__ = ["basename", "csvmod", "ensure_str", "exists", "expanduser", "isfile", "isdir", "join", "json",
+           "literal_eval", "makedirs", "re", "remove", "splitext", "text_or_path", "BytesIO", "StringIO",
+           "CSV_DELIMITER", "DEFAULT_EXCL", "INMEMORY", "META_EXCL", "MISSING_TOKEN", "PYARROW", "TARGET_NAME"]
+if PYARROW:
+    __all__ += ["feather", "orc", "pandas", "parquet", "pyarrow"]
+
+
+CSV_DELIMITER = ";"
+DEFAULT_EXCL  = ("hash", "realpath", "format", "size", "ctime", "mtime")  # origin: executables used in the Packing Box
+INMEMORY      = "<memory>"
+META_EXCL     = ["created", "modified", "revision"]
+MISSING_TOKEN = "?"
+TARGET_NAME   = "label"
+
+
+def ensure_str(string):
+    try:
+        return string.decode()
+    except AttributeError:
+        return str(string)
+
+
+def text_or_path(f):
+    def _wrapper(txt_path, *args, **kwargs):
+        if not isinstance(txt_path, bytes):
+            if isinstance(txt_path, str):
+                if not re.search(r"[<>:\'\"/\\|?*\x00-\x1f]", txt_path, re.I) and exists(txt_path):
+                    if not isdir(txt_path):
+                        with open(txt_path, "rb") as fh:
+                            txt_path = fh.read()
+                else:
+                    txt_path = txt_path.encode()
+            else:
+                raise ValueError(f"Bad input text or path ({type(txt_path)})")
+        return f(txt_path, *args, **kwargs)
+    return _wrapper
+

{dsff-1.2.0 → dsff-1.2.3}/src/dsff/formats/__init__.py

@@ -22,6 +22,9 @@ if PYARROW:  # pragma: no cover
 
 
 __all__ = ["DSFF"]
+for k in list(globals().keys()):
+    if k.startswith("is_") or k.startswith("load_"):
+        __all__.append(k)
 
 _FORMAT_TEXT_ALIAS = {'arff': "ARFF", 'csv': "CSV", 'db': "SQL", 'orc': "ORC"}
 _NO_EXT_FORMAT     = ["dataset"]
@@ -343,4 +346,16 @@ class DSFF:
         if p != INMEMORY and not p.endswith(".dsff"):
             p += ".dsff"
         return p
+    
+    @staticmethod
+    @text_or_path
+    def format(text):
+        for k, func in globals().items():
+            if k.startswith("is_") and func(text):
+                return k.split("_", 1)[1]
+    
+    @staticmethod
+    @text_or_path
+    def is_valid_format(text):
+        return DSFF.format(text) is not None
 

{dsff-1.2.0 → dsff-1.2.3}/src/dsff/formats/arff.py

@@ -2,68 +2,67 @@
 from .__common__ import *
 
 
-__all__ = ["from_arff", "to_arff"]
+__all__ = ["from_arff", "is_arff", "load_arff", "to_arff"]
 
 
-def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
-    """ Populate the DSFF file from an ARFF file. """
-    d, features = [], {}
-    with open(path) as f:
-        relation, attributes, data = False, [False, False], False
-        for n, l in enumerate(f, 1):
-            l, pf = l.strip(), f"Line {n}: "
-            # the file shall start with "@RELATION"
-            if not relation:
-                if l.startswith("@RELATION "):
-                    relation = True
-                    try:
-                        dsff['title'] = re.match(r"@RELATION\s+('[^']*'|\"[^\"]*\")$", l).group(1).strip("'\"")
-                        continue
-                    except Exception as e:
-                        raise BadInputData(f"{pf}failed on @RELATION ({e})")
-                else:
-                    raise BadInputData(f"{pf}did not find @RELATION")
-            # get metadata and feature descriptions from comments
-            if l.startswith("%"):
-                if re.match(r"^\%\s+metadata\s*\:\s*\{.*\}$", l):
-                    dsff.write(metadata=literal_eval(l.split(":", 1)[1]))
-                elif (m := re.match(r"^\%\s+(.*?)\s*\:\s*(.*?)$", l)):
-                    name, descr = m.groups()
-                    features[name] = descr
-                continue
-            # then ignore blank lines
-            if l == "":
-                if attributes[0] and not attributes[1]:
-                    # close the atributes block
-                    attributes[1] = True
-                    n_cols = len(d[0])
-                continue
-            if l.startswith("@ATTRIBUTE "):
-                if not attributes[0]:
-                    attributes[0] = True
-                if len(d) == 0:
-                    # start the attributes block
-                    d.append([])
-                if attributes[1]:
-                    raise BadInputData(f"{pf}found @ATTRIBUTE out of the attributes block)")
+def _parse(text_or_fh, target=TARGET_NAME, missing=MISSING_TOKEN, **kw):
+    d, features, metadata, title = [], {}, {}, ""
+    relation, attributes, data = False, [False, False], False
+    for n, l in enumerate(t.splitlines() if isinstance(t := text_or_fh, str) else t):
+        l, pf = l.strip(), f"Line {n}: "
+        # the file shall start with "@RELATION"
+        if not relation:
+            if l.upper().startswith("@RELATION "):
+                relation = True
                 try:
-                    header = re.match(r"@ATTRIBUTE\s+([^\s]+)\s+[A-Z]+$", l).group(1)
-                    if header == "class":
-                        header = target
-                    d[0].append(header)
-                    continue
-                except AttributeError:
-                    raise BadInputData(f"{pf}failed on @ATTRIBUTE (bad type)")
-            if not data:
-                if l == "@DATA":
-                    data = True
+                    title = re.match(r"@RELATION\s+('[^']*'|\"[^\"]*\")$", l, re.I).group(1).strip("'\"")
                     continue
+                except Exception as e:
+                    raise BadInputData(f"{pf}failed on @RELATION ({e})")
+            else:
+                raise BadInputData(f"{pf}did not find @RELATION")
+        # get metadata and feature descriptions from comments
+        if l.startswith("%"):
+            if re.match(r"^\%\s+metadata\s*\:\s*\{.*\}$", l):
+                metadata = literal_eval(l.split(":", 1)[1])
+            elif (m := re.match(r"^\%\s+(.*?)\s*\:\s*(.*?)$", l)):
+                name, descr = m.groups()
+                features[name] = descr
+            continue
+        # then ignore blank lines
+        if l == "":
+            if attributes[0] and not attributes[1]:
+                # close the atributes block
+                attributes[1] = True
+            continue
+        if l.upper().startswith("@ATTRIBUTE "):
+            if not attributes[0]:
+                attributes[0] = True
+            if len(d) == 0:
+                # start the attributes block
+                d.append([])
+            if attributes[1]:
+                raise BadInputData(f"{pf}found @ATTRIBUTE out of the attributes block)")
+            try:
+                header = re.match(r"@ATTRIBUTE\s+([^\s]+)\s+(?:[a-zA-Z]+|\{.*?\})$", l, re.I).group(1).strip("'\"")
+                if header == "class":
+                    header = target
                 else:
-                    raise BadInputData(f"{pf}did not find @DATA where expected")
-            row = list(map(lambda x: x.strip("'\""), re.split(r",\s+", l)))
-            if len(row) != n_cols:
-                raise BadInputData(f"{pf}this row does not match the number of columns")
-            d.append(row)
+                    features.setdefault(header, "")
+                d[0].append(header)
+                continue
+            except AttributeError:
+                raise BadInputData(f"{pf}failed on @ATTRIBUTE (bad type)")
+        if not data:
+            if l.upper() == "@DATA":
+                data = True
+                n_cols = len(d[0])
+                continue
+            else:
+                raise BadInputData(f"{pf}did not find @DATA where expected")
+        if len(row := list(map(lambda x: x.strip("'\""), re.split(r",\s*", l)))) != n_cols:
+            raise BadInputData(f"{pf}this row does not match the number of columns")
+        d.append(row)
     for i in range(n_cols):
         values = []
         for j, row in enumerate(d):
@@ -76,12 +75,36 @@ def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN):
         if "".join(sorted(set(values))) in ["0", "1", "01"]:
             for j, row in enumerate(d):
                 if j > 0:
-                    row[i] = {'0': "False", '1': "True"}[row[i]]
-    dsff.write(data=d)
-    dsff.write(features=features)
+                    row[i] = {'0': "False", '1': "True"}[row[i]]    
+    return d, features, metadata, title
+
+
+def from_arff(dsff, path=None, target=TARGET_NAME, missing=MISSING_TOKEN, **kw):
+    """ Populate the DSFF file from an ARFF file. """
+    with open(path) as f:
+        d, ft, md, t = _parse(f, target, missing)
+    dsff['title'] = t
+    dsff.write(data=d, features=ft, metadata=md)
+
+
+@text_or_path
+def is_arff(text, target=TARGET_NAME, missing=MISSING_TOKEN, **kw):
+    """ Check if the input text or path is a valid ARFF. """
+    try:
+        _parse(ensure_str(text))
+        return True
+    except (BadInputData, UnicodeDecodeError):
+        return False
+
+
+def load_arff(path, target=TARGET_NAME, missing=MISSING_TOKEN, **kw):
+    """ Load an ARFF file as a dictionary with data, features and metadata. """
+    with open(path) as f:
+        d, ft, md, _ = _parse(f, target, missing)
+    return {'data': d, 'features': ft, 'metadata': md}
 
 
-def to_arff(dsff, path=None, target=TARGET_NAME, exclude=DEFAULT_EXCL, missing=MISSING_TOKEN, text=False):
+def to_arff(dsff, path=None, target=TARGET_NAME, exclude=DEFAULT_EXCL, missing=MISSING_TOKEN, text=False, **kw):
     """ Output the dataset in ARFF format, suitable for use with the Weka framework, saved as a file or output as a
          string. """
     name = splitext(basename(path))[0]

dsff-1.2.3/src/dsff/formats/csv.py

@@ -0,0 +1,41 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import *
+
+
+__all__ = ["from_csv", "is_csv", "load_csv", "to_csv"]
+
+
+def from_csv(dsff, path=None, exclude=DEFAULT_EXCL, **kw):
+    """ Populate the DSFF file from a CSV file. """
+    dsff.write(**load_csv(path))
+
+
+@text_or_path
+def is_csv(text, **kw):
+    """ Check if the input text or path is a valid CSV. """
+    try:
+        dialect = csvmod.Sniffer().sniff(text := ensure_str(text))
+        csvmod.Sniffer().has_header(text)
+        return True
+    except (csvmod.Error, UnicodeDecodeError):
+        return False
+
+
+def load_csv(path, exclude=DEFAULT_EXCL, **kw):
+    """ Load a CSV file as a dictionary with data, features and metadata. """
+    data = {'metadata': {}}
+    with open(expanduser(path)) as f:
+        data['data'] = [r for r in csvmod.reader(f, delimiter=CSV_DELIMITER)]
+    data['features'] = {h: "" for h in data['data'][0] if h not in exclude}
+    return data
+
+
+def to_csv(dsff, path=None, text=False, **kw):
+    """ Create a CSV from the data worksheet, saved as a file or output as a string. """
+    with (StringIO() if text else open(path, 'w+')) as f:
+        writer = csvmod.writer(f, delimiter=";")
+        for row in dsff.data:
+            writer.writerow(row)
+        if text:
+            return f.getvalue()
+

dsff-1.2.3/src/dsff/formats/dataset.py

@@ -0,0 +1,67 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import *
+from .csv import load_csv
+
+
+__all__ = ["from_dataset", "is_dataset", "load_dataset", "to_dataset"]
+
+
+def _parse(path):
+    if not isdir(expanduser(path)):
+        raise BadInputData("Not a folder")
+    if len(missing := [f for f in ["data.csv", "features.json", "metadata.json"] if not isfile(join(path, f))]) > 0:
+        raise BadInputData(f"Not a valid dataset folder (missing: {', '.join(missing)})")
+
+
+def from_dataset(dsff, path=None, **kw):
+    """ Populate the DSFF file from a Dataset structure. """
+    _parse(path)
+    dsff.write(path)
+
+
+def is_dataset(path, **kw):
+    """ Check if the input path is a valid Dataset. """
+    try:
+        _parse(path)
+        return True
+    except BadInputData:
+        return False
+
+
+def load_dataset(path, **kw):
+    """ Load a dataset folder as a dictionary with data, features and metadata. """
+    if not isdir(d := expanduser(str(path))):
+        raise BadInputData("Not a folder")
+    dp, fp, mp = join(d, "data.csv"), join(d, "features.json"), join(d, "metadata.json")
+    data = {}
+    data['data'] = load_csv(dp)['data']
+    with open(fp) as f:
+        data['features'] = json.load(f)
+    with open(mp) as f:
+        data['metadata'] = json.load(f)
+    return data
+
+
+def to_dataset(dsff, path=None, **kw):
+    """ Create a dataset folder according to the following structure ;
+    name
+     +-- data.csv
+     +-- features.json
+     +-- metadata.json
+    """
+    makedirs(path, exist_ok=True)
+    # handle data first
+    dsff.logger.debug("> making data.csv...")
+    with open(join(path, "data.csv"), 'w+') as f:
+        writer = csvmod.writer(f, delimiter=";")
+        for cells in dsff['data'].rows:
+            writer.writerow([dsff._DSFF__eval(c.value) for c in cells])
+    # then handle features dictionary
+    dsff.logger.debug("> making features.json...")
+    with open(join(path, "features.json"), 'w+') as f:
+        json.dump(dsff.features, f, indent=2)
+    # finally handle metadata dictionary
+    dsff.logger.debug("> making metadata.json...")
+    with open(join(path, "metadata.json"), 'w+') as f:
+        json.dump(dsff.metadata, f, indent=2)
+

{dsff-1.2.0 → dsff-1.2.3}/src/dsff/formats/db.py

@@ -2,15 +2,51 @@
 from .__common__ import *
 
 
-__all__ = ["from_db", "to_db"]
+__all__ = ["from_db", "is_db", "load_db", "to_db"]
 
 
-def from_db(dsff, path=None, exclude=DEFAULT_EXCL):
+def from_db(dsff, path=None, **kw):
     """ Populate the DSFF file from a SQLDB file. """
+    dsff.write(**load_db(path))
+
+
+@text_or_path
+def is_db(data, **kw):
+    """ Check if the input data or path is a valid SQL database. """
+    from sqlite3 import connect, Error
+    from sys import version_info
+    if not data.startswith(b"SQLite format 3\x00"):
+        return False
+    if version_info.minor > 10:
+        try:
+            with connect(":memory:") as c:
+                c.deserialize(data)
+                c.execute("PRAGMA schema_version;")
+            return True
+        except Error:
+            return False
+    else:  # pragma: no cover
+        from os import close, write
+        from tempfile import mkstemp
+        fd, path = mkstemp(suffix=".db")
+        try:
+            write(fd, data)
+            close(fd)
+            fd = None
+            with connect(f"file:{path}?mode=ro&immutable=1", uri=True) as c:
+                c.execute("PRAGMA schema_version;").fetchone()
+            return True
+        except (Error, OSError):
+            return False
+
+
+def load_db(path, **kw):
+    """ Load a SQLDB file as a dictionary with data, features and metadata. """
     from json import loads
+    from os.path import basename, splitext
     from sqlite3 import connect
     conn = connect(path)
-    cursor = conn.cursor()
+    cursor, data = conn.cursor(), {}
     # list tables
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
     tables = [table[0] for table in cursor.fetchall()]
@@ -20,17 +56,18 @@ def from_db(dsff, path=None, exclude=DEFAULT_EXCL):
     cursor.execute("PRAGMA table_info('data')")
     headers = [[col[1] for col in cursor.fetchall()]]
     cursor.execute("SELECT * FROM data;")
-    dsff.write(headers + [r for r in cursor.fetchall()])
+    data['data'] = headers + [r for r in cursor.fetchall()]
     # import feature definitions
     cursor.execute("SELECT name,description FROM features;")
-    dsff.write(features={r[0]: r[1] for r in cursor.fetchall()})
+    data['features'] = {r[0]: r[1] for r in cursor.fetchall()}
     # import metadata
     cursor.execute("SELECT key,value FROM metadata;")
-    dsff.write(metadata={r[0]: loads(r[1]) if isinstance(r[1], str) else r[1] for r in cursor.fetchall()})
+    data['metadata'] = {r[0]: loads(r[1]) if isinstance(r[1], str) else r[1] for r in cursor.fetchall()}
     conn.close()
+    return data
 
 
-def to_db(dsff, path=None, text=False, primary_index=0):
+def to_db(dsff, path=None, text=False, primary_index=0, **kw):
     """ Create a SQLDB from the data worksheet, saved as a file or output as a string. """
     from json import dumps
     from sqlite3 import connect

dsff-1.2.3/src/dsff/formats/pa.py

@@ -0,0 +1,46 @@
+# -*- coding: UTF-8 -*-
+from .__common__ import *
+
+
+__all__ = []
+
+
+def _nowrite(m):
+    raise NotImplementedError(f"none of {m}.write_table and {m}.write_{m} is implemented")
+
+
+def _parse(ds):
+    return {
+        'data': [ds.schema.names] + [list(r.values()) for r in ds.to_pylist()],
+        'features': {k.decode(): v.decode() for k, v in ds.schema.metadata.items()},
+        'metadata': literal_eval(ds.schema.metadata.pop(b'__metadata__', b"{}").decode()),
+    }
+
+
+for module in ["feather", "orc", "parquet"]:
+    __all__ += [f"from_{module}", f"is_{module}", f"load_{module}", f"to_{module}"]
+    def gen_func(m):
+        def from_(dsff, path=None, **kw):
+            from os.path import basename, splitext
+            dsff.write(**_parse(globals()[m].read_table(path)))
+        from_.__name__ = f"from_{m}"
+        def is_(data, **kw):
+            try:
+                globals()[m].read_table(pyarrow.BufferReader(data))
+                return True
+            except Exception:
+                return False
+        is_.__name__ = f"is_{m}"
+        def load_(path, **kw):
+            return _parse(globals()[m].read_table(path))
+        load_.__name__ = f"load_{m}"
+        def to_(dsff, path=None, text=False, **kw):
+            with (BytesIO() if text else open(path, 'wb+')) as f:
+                getattr(globals()[m], "write_table", getattr(globals()[m], f"write_{m}", _nowrite))(dsff._to_table(), f)
+                if text:
+                    return f.getvalue()
+        to_.__name__ = f"to_{m}"
+        return from_, text_or_path(is_), load_, to_
+    globals()[f'from_{module}'], globals()[f'is_{module}'], globals()[f'load_{module}'], globals()[f'to_{module}'] = \
+        gen_func(module)
+

{dsff-1.2.0 → dsff-1.2.3/src/dsff.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dsff
-Version: 1.2.0
+Version: 1.2.3
 Summary: DataSet File Format (DSFF)
 Author-email: Alexandre D'Hondt <alexandre.dhondt@gmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -696,8 +696,10 @@ Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: openpyxl
-Provides-Extra: extra
-Requires-Dist: pyarrow; extra == "extra"
+Provides-Extra: all
+Requires-Dist: dsff[arrow]; extra == "all"
+Provides-Extra: arrow
+Requires-Dist: pyarrow; extra == "arrow"
 Dynamic: license-file
 
 <p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/imgs/logo.png"></p>
@@ -738,9 +740,34 @@ pip install --user dsff
 ```python
 >>> import dsff
 >>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
-    f.to_dataset()  # creates ./[dsff-title] with data.csv, features.json and metadata.json
+    f.to_dataset()  # creates ./my-dataset with data.csv, features.json and metadata.json
 ```
 
+**
+
+## Extensions
+
+**Install all available extensions**
+
+```sh
+pip install --user dsff[all]
+```
+
+**Dealing with [Apache Arrow](https://arrow.apache.org/) formats**
+
+```sh
+pip install --user dsff[arrow]
+```
+
+```python
+>>> import dsff
+>>> with dsff.DSFF("/path/to/my-dataset.dsff") as f:
+    f.to_feather()  # creates ./my-dataset.feather
+    f.to_orc()      # creates ./my-dataset.orc
+    f.to_parquet()  # creates ./my-dataset.parquet
+```
+
+
 ## Related Projects
 
 You may also like these:

dsff-1.2.3/src/dsff.egg-info/requires.txt

@@ -0,0 +1,7 @@
+openpyxl
+
+[all]
+dsff[arrow]
+
+[arrow]
+pyarrow

{dsff-1.2.0 → dsff-1.2.3}/tests/test_dsff.py

@@ -76,6 +76,7 @@ class TestDsff(TestCase):
             TEST_METADATA[k] = v
     
     def test_conversion_arff(self):
+        self.assertTrue(is_arff(TEST_ARFF))
         # DSFF to ARFF
         create_test_dsff()
         with DSFF(TEST) as f:
@@ -84,10 +85,11 @@ class TestDsff(TestCase):
         with DSFF(mode='w+') as f:
             self.assertRaises(ValueError, f.to_arff, ())
         # ARFF to DSFF
-        arff = TEST_BASENAME + ".arff"
         #  use a test ARFF with a comment line
-        with open(arff, 'w') as f:
+        with open(arff := f"{TEST_BASENAME}.arff", 'w') as f:
             f.write(TEST_ARFF)
+        self.assertTrue(is_arff(arff))
+        self.assertIsInstance(load_arff(arff), dict)
         with DSFF() as f:
             f.from_arff(TEST_BASENAME)
         # test for multiple error scenarios
@@ -123,25 +125,18 @@ class TestDsff(TestCase):
             d.insert(10, d.pop(6))
     
     def test_conversion_csv(self):
+        self.assertFalse(is_csv(b"PK\x03\x04\x14\x00\x00\x00\x08\x00P\xb3T\\F\xc7MH"))
         # DSFF to CSV
         create_test_dsff()
         with DSFF(TEST) as f:
             self.assertIsNotNone(f.to_csv(text=True))
             self.assertIsNone(f.to_csv())
+        self.assertTrue(is_csv(csv := f"{TEST_BASENAME}.csv"))
+        self.assertIsInstance(load_csv(csv), dict)
         # CSV to DSFF
         with DSFF() as f:
             f.from_csv(TEST_BASENAME)
     
-    def test_conversion_db(self):
-        # DSFF to SQL database
-        create_test_dsff()
-        with DSFF(TEST) as f:
-            self.assertIsNotNone(f.to_db(text=True))
-            self.assertIsNone(f.to_db())
-        # SQL database to DSFF
-        with DSFF() as f:
-            f.from_db(TEST_BASENAME)
-    
     def test_conversion_dataset(self):
         # DSFF to FilelessDataset
         create_test_dsff()
@@ -150,6 +145,9 @@ class TestDsff(TestCase):
         # FilelessDataset to DSFF
         with DSFF() as f:
             f.from_dataset(TEST_BASENAME)
+        self.assertTrue(is_dataset(TEST_BASENAME))
+        self.assertRaises(BadInputData, load_dataset, 0)
+        self.assertIsInstance(load_dataset(TEST_BASENAME), dict)
         # FilelessDataset to DSFF (bad input dataset)
         os.remove(os.path.join(TEST_BASENAME, "metadata.json"))
         with DSFF() as f:
@@ -161,6 +159,23 @@ class TestDsff(TestCase):
         with DSFF() as f:
             self.assertRaises(BadInputData, f.from_dataset, TEST_BASENAME)
         os.remove(TEST_BASENAME)
+        os.makedirs(TEST_BASENAME)
+        self.assertFalse(is_dataset(TEST_BASENAME))
+        rmdir(TEST_BASENAME)
+    
+    def test_conversion_db(self):
+        self.assertFalse(is_db(b"PK\x03\x04\x14\x00\x00\x00\x08\x00P\xb3T\\F\xc7MH"))
+        self.assertFalse(is_db(b"SQLite format 3\x00\x10\x00\x01\x01\x00@"))
+        # DSFF to SQL database
+        create_test_dsff()
+        with DSFF(TEST) as f:
+            self.assertIsNotNone(f.to_db(text=True))
+            self.assertIsNone(f.to_db())
+        self.assertTrue(is_db(db := f"{TEST_BASENAME}.db"))
+        self.assertIsInstance(load_db(db), dict)
+        # SQL database to DSFF
+        with DSFF() as f:
+            f.from_db(TEST_BASENAME)
     
     def test_conversion_pyarrow_formats(self):
         if PYARROW:
@@ -172,13 +187,17 @@ class TestDsff(TestCase):
                     f.from_dataset(path=TEST_BASENAME)
                     self.assertIsNone(getattr(f, f"to_{fmt}")(TEST_BASENAME))
                     self.assertIsNotNone(getattr(f, f"to_{fmt}")(text=True))
+                is_ = globals()[f'is_{fmt}']
+                self.assertTrue(is_(fn := f"{TEST_BASENAME}.{fmt}"))
+                self.assertFalse(is_(b"PK\x03\x04\x14\x00\x00\x00\x08\x00P\xb3T\\F\xc7MH"))
+                self.assertIsInstance(globals()[f'load_{fmt}'](fn), dict)
                 with DSFF(INMEMORY) as f:
                     self.assertIsNone(getattr(f, f"from_{fmt}")(TEST_BASENAME))
                     f.to_dataset(path=TEST_BASENAME)
                 os.remove(f"{TEST_BASENAME}.{fmt}")
             rmdir(TEST_BASENAME)
             # test without the 'label' column
-            with open(TEST_BASENAME + ".arff", 'w') as f:
+            with open(f"{TEST_BASENAME}.arff", 'w') as f:
                 f.write(TEST_ARFF.replace("class", "test").replace("?", "-1"))
             with DSFF() as f:
                 f.from_arff(TEST_BASENAME)
@@ -190,10 +209,13 @@ class TestDsff(TestCase):
             self.assertIsNone(f.to_dataset())
             f.name = TEST
             self.assertIsNotNone(f.headers)
+        self.assertTrue(DSFF.is_valid_format(TEST_BASENAME))
         with DSFF(INMEMORY) as f:
             self.assertRaises(EmptyDsffFile, f.to_arff)
             self.assertIsNone(f.from_dataset(TEST_BASENAME))
             self.assertIsNone(f.to_arff())
+        self.assertRaises(ValueError, DSFF.is_valid_format, 0)
+        self.assertTrue(DSFF.is_valid_format(f"undefined.arff"))
         os.remove(f"undefined.arff")
         rmdir(TEST_BASENAME)
         with DSFF(INMEMORY) as f:

dsff-1.2.0/docs/coverage.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="114" height="20" role="img" aria-label="coverage: 89.26%"><title>coverage: 89.26%</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="114" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="61" height="20" fill="#555"/><rect x="61" width="53" height="20" fill="#97ca00"/><rect width="114" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="315" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="510">coverage</text><text x="315" y="140" transform="scale(.1)" fill="#fff" textLength="510">coverage</text><text aria-hidden="true" x="865" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">89.26%</text><text x="865" y="140" transform="scale(.1)" fill="#fff" textLength="430">89.26%</text></g></svg>

dsff-1.2.0/src/dsff/VERSION.txt

@@ -1 +0,0 @@
-1.2.0

dsff-1.2.0/src/dsff/formats/__common__.py

@@ -1,33 +0,0 @@
-# -*- coding: UTF-8 -*-
-import csv as csvmod
-import json
-import re
-from ast import literal_eval
-from io import BytesIO, StringIO
-from os import makedirs, remove
-from os.path import basename, expanduser, isfile, isdir, join, splitext
-try:  # pragma: no cover
-    import pyarrow
-    import pyarrow.feather as feather
-    import pyarrow.orc as orc
-    import pyarrow.parquet as parquet
-    import pandas
-    PYARROW = True
-except ImportError:  # pragma: no cover
-    PYARROW = False
-
-
-__all__ = ["basename", "csvmod", "expanduser", "isfile", "isdir", "join", "json", "literal_eval", "makedirs",
-           "remove", "splitext", "re", "BytesIO", "StringIO",
-           "CSV_DELIMITER", "DEFAULT_EXCL", "INMEMORY", "META_EXCL", "MISSING_TOKEN", "PYARROW", "TARGET_NAME"]
-if PYARROW:
-    __all__ += ["feather", "orc", "pandas", "pyarrow", "parquet"]
-
-
-CSV_DELIMITER = ";"
-DEFAULT_EXCL  = ("hash", "realpath", "format", "size", "ctime", "mtime")  # origin: executables used in the Packing Box
-INMEMORY      = "<memory>"
-META_EXCL     = ["created", "modified", "revision"]
-MISSING_TOKEN = "?"
-TARGET_NAME   = "label"
-

dsff-1.2.0/src/dsff/formats/csv.py

@@ -1,29 +0,0 @@
-# -*- coding: UTF-8 -*-
-from .__common__ import *
-
-
-__all__ = ["from_csv", "to_csv"]
-
-
-def from_csv(dsff, path=None, exclude=DEFAULT_EXCL):
-    """ Populate the DSFF file from a CSV file. """
-    dsff.write(path)
-    features = {}
-    for headers in dsff['data'].rows:
-        for header in headers:
-            if header.value in exclude:
-                continue
-            features[header.value] = ""
-        break
-    dsff.write(features=features)
-
-
-def to_csv(dsff, path=None, text=False):
-    """ Create a CSV from the data worksheet, saved as a file or output as a string. """
-    with (StringIO() if text else open(path, 'w+')) as f:
-        writer = csvmod.writer(f, delimiter=";")
-        for row in dsff.data:
-            writer.writerow(row)
-        if text:
-            return f.getvalue()
-

dsff-1.2.0/src/dsff/formats/dataset.py

@@ -1,40 +0,0 @@
-# -*- coding: UTF-8 -*-
-from .__common__ import *
-
-
-__all__ = ["from_dataset", "to_dataset"]
-
-
-def from_dataset(dsff, path=None):
-    """ Populate the DSFF file from a Dataset structure. """
-    if not isdir(path):
-        raise BadInputData("Not a folder")
-    else:
-        if len(missing := [f for f in ["data.csv", "features.json", "metadata.json"] if not isfile(join(path, f))]) > 0:
-            raise BadInputData(f"Not a valid dataset folder (missing: {', '.join(missing)})")
-    dsff.write(path)
-
-
-def to_dataset(dsff, path=None):
-    """ Create a dataset folder according to the following structure ;
-    name
-     +-- data.csv
-     +-- features.json
-     +-- metadata.json
-    """
-    makedirs(path, exist_ok=True)
-    # handle data first
-    dsff.logger.debug("> making data.csv...")
-    with open(join(path, "data.csv"), 'w+') as f:
-        writer = csvmod.writer(f, delimiter=";")
-        for cells in dsff['data'].rows:
-            writer.writerow([dsff._DSFF__eval(c.value) for c in cells])
-    # then handle features dictionary
-    dsff.logger.debug("> making features.json...")
-    with open(join(path, "features.json"), 'w+') as f:
-        json.dump(dsff.features, f, indent=2)
-    # finally handle metadata dictionary
-    dsff.logger.debug("> making metadata.json...")
-    with open(join(path, "metadata.json"), 'w+') as f:
-        json.dump(dsff.metadata, f, indent=2)
-

dsff-1.2.0/src/dsff/formats/pa.py

@@ -1,29 +0,0 @@
-# -*- coding: UTF-8 -*-
-from .__common__ import *
-
-
-__all__ = []
-
-
-def _nowrite(m):
-    raise NotImplementedError(f"none of {m}.write_table and {m}.write_{m} is implemented")
-
-
-for module in ["feather", "orc", "parquet"]:
-    __all__ += [f"from_{module}", f"to_{module}"]
-    def gen_func(m):
-        def from_(dsff, path=None, exclude=DEFAULT_EXCL):
-            dataset = globals()[m].read_table(path)
-            dsff.write(data=[dataset.schema.names] + [list(r.values()) for r in dataset.to_pylist()],
-                       metadata=literal_eval(dataset.schema.metadata.pop(b'__metadata__', b"{}").decode()),
-                       features={k.decode(): v.decode() for k, v in dataset.schema.metadata.items()})
-        from_.__name__ = f"from_{m}"
-        def to_(dsff, path=None, text=False):
-            with (BytesIO() if text else open(path, 'wb+')) as f:
-                getattr(globals()[m], "write_table", getattr(globals()[m], f"write_{m}", _nowrite))(dsff._to_table(), f)
-                if text:
-                    return f.getvalue()
-        to_.__name__ = f"to_{m}"
-        return from_, to_
-    globals()[f'from_{module}'], globals()[f'to_{module}'] = gen_func(module)
-

dsff-1.2.0/src/dsff.egg-info/requires.txt

@@ -1,4 +0,0 @@
-openpyxl
-
-[extra]
-pyarrow