mlrl-testbed-arff 0.12.0__py3-none-any.whl

mlrl/testbed_arff/experiments/input/sources/__init__.py

@@ -0,0 +1,6 @@
+"""
+Author Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow to read input data from ARFF files.
+"""
+from mlrl.testbed_arff.experiments.input.sources.source_arff import ArffFileSource

mlrl/testbed_arff/experiments/input/sources/source_arff.py

@@ -0,0 +1,239 @@
+"""
+Author Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow reading datasets from ARFF files.
+"""
+import logging as log
+
+from functools import cached_property
+from os import path
+from typing import Any, List, Optional, Set
+from xml.dom import minidom
+
+import arff
+import numpy as np
+
+from scipy.sparse import coo_array, csc_array, sparray
+
+from mlrl.testbed_arff.experiments.output.sinks.sink_arff import ArffFileSink
+
+from mlrl.testbed_sklearn.experiments.dataset import Attribute, AttributeType, TabularDataset
+
+from mlrl.testbed.experiments.dataset import Dataset
+from mlrl.testbed.experiments.input.data import DatasetInputData
+from mlrl.testbed.experiments.input.sources.source import DatasetFileSource
+from mlrl.testbed.experiments.state import ExperimentState
+from mlrl.testbed.util.io import open_readable_file
+
+
+def normalize_attribute_name(name: str) -> str:
+    """
+    Normalizes the name of an attribute by removing forbidden characters.
+
+    :param name:    The name to be normalized
+    :return:        The normalized name
+    """
+    name = name.strip()
+    if name.startswith('\'') or name.startswith('"'):
+        name = name[1:]
+    if name.endswith('\'') or name.endswith('"'):
+        name = name[:(len(name) - 1)]
+    return name.replace('\\\'', '\'').replace('\\"', '"')
+
+
+class ArffFileSource(DatasetFileSource):
+    """
+    Allows to read a dataset from an ARFF file.
+    """
+
+    class ArffFile:
+        """
+        Provides access to the content of an ARFF file.
+        """
+
+        def __init__(self, matrix: sparray, arff_attributes: List[Any], relation: str):
+            """
+            param matrix:           The data matrix that is stored in the file
+            param arff_attributes:  The attributes defined in the file
+            param relation:         The @relation declaration contained in the file
+            """
+            self.matrix = matrix
+            self.arff_attributes = arff_attributes
+            self.relation = relation
+
+        @staticmethod
+        def from_file(file_path: str, sparse: bool, dtype: np.dtype) -> 'ArffFileSource.ArffFile':
+            """
+            Loads the content of an ARFF file.
+
+            :param file_path:   The path to the ARFF file
+            :param sparse:      True, if the ARFF file is given in sparse format, False otherwise. If the given format
+                                is incorrect, an `arff.BadLayout` is raised
+            :param dtype:       The type of the data matrix to be read from the file
+            :return:            A dictionary that stores the content of the ARFF file
+            """
+            with open_readable_file(file_path) as arff_file:
+                arff_dict = arff.load(arff_file, encode_nominal=True, return_type=arff.COO if sparse else arff.DENSE)
+
+                if sparse:
+                    data = arff_dict['data']
+                    values = data[0]
+                    row_indices = data[1]
+                    col_indices = data[2]
+                    shape = (max(row_indices) + 1, max(col_indices) + 1)
+                    matrix = coo_array((values, (row_indices, col_indices)), shape=shape, dtype=dtype)
+                    matrix = matrix.tocsc()
+                else:
+                    data = arff_dict['data']
+                    matrix = csc_array(data, dtype=dtype)
+
+            attributes = arff_dict['attributes']
+            relation = arff_dict['relation']
+            return ArffFileSource.ArffFile(matrix, arff_attributes=attributes, relation=relation)
+
+        @cached_property
+        def attributes(self) -> List[Attribute]:
+            """
+            A list that contains all attributes defined in the ARFF file.
+            """
+            attributes = []
+
+            for attribute in self.arff_attributes:
+                attribute_name = normalize_attribute_name(attribute[0])
+                type_definition = attribute[1]
+
+                if isinstance(type_definition, list):
+                    attribute_type = AttributeType.NOMINAL
+                    nominal_values = type_definition
+                else:
+                    type_definition = str(type_definition).lower()
+                    nominal_values = None
+
+                    if type_definition == 'integer':
+                        attribute_type = AttributeType.ORDINAL
+                    elif type_definition in ('real', 'numeric'):
+                        attribute_type = AttributeType.NUMERICAL
+                    else:
+                        raise ValueError('Encountered unsupported attribute type: ' + type_definition)
+
+                attribute = Attribute(name=attribute_name, attribute_type=attribute_type, nominal_values=nominal_values)
+                attributes.append(attribute)
+
+            return attributes
+
+    class ArffDataset:
+        """
+        Provides access to the content of an ARFF file and the corresponding Mulan XML file, if available.
+        """
+
+        def __parse_output_names_from_relation(self) -> Set[str]:
+            parameter_name = '-C '
+            arff_file = self.arff_file
+            relation = arff_file.relation
+            index = relation.index(parameter_name)
+            parameter_value = relation[index + len(parameter_name):]
+            index = parameter_value.find(' ')
+
+            if index >= 0:
+                parameter_value = parameter_value[:index]
+
+            num_outputs = int(parameter_value)
+            attributes = arff_file.attributes
+            return {normalize_attribute_name(attributes[i].name) for i in range(num_outputs)}
+
+        def __init__(self, arff_file: 'ArffFileSource.ArffFile', output_names: Optional[Set[str]]):
+            """
+            :param arff_file:       The content of the ARFF file
+            :param output_names:    The names of all outputs contained in the dataset
+            """
+            self.arff_file = arff_file
+            self.output_names = output_names if output_names else self.__parse_output_names_from_relation()
+
+        @staticmethod
+        def from_file(arff_file: 'ArffFileSource.ArffFile', file_path: str) -> 'ArffFileSource.ArffDataset':
+            """
+            Creates and returns an ARFF dataset from given ARFF file and a corresponding Mulan XML file, if available.
+
+
+            :param arff_file:       The content of the ARFF file
+            :param file_path:       The path to the XML file
+            :return:                The ARFF dataset that has been created
+            """
+            if path.isfile(file_path):
+                log.debug('Parsing meta-data from file \"%s\"...', file_path)
+                xml_doc = minidom.parse(file_path)
+                tags = xml_doc.getElementsByTagName('label')
+                output_names = {normalize_attribute_name(tag.getAttribute('name')) for tag in tags}
+            else:
+                output_names = None
+                log.debug(
+                    'Mulan XML file \"%s\" does not exist. If possible, information about the dataset\'s outputs is '
+                    + 'parsed from the ARFF file\'s @relation declaration as intended by the MEKA dataset format...',
+                    file_path)
+
+            return ArffFileSource.ArffDataset(arff_file=arff_file, output_names=output_names)
+
+        @cached_property
+        def features(self) -> List[Attribute]:
+            """
+            A list that stores all features contained in the dataset.
+            """
+            return [attribute for attribute in self.arff_file.attributes if attribute.name not in self.output_names]
+
+        @cached_property
+        def outputs(self) -> List[Attribute]:
+            """
+            A list that stores all outputs contained in the dataset.
+            """
+            return [attribute for attribute in self.arff_file.attributes if attribute.name in self.output_names]
+
+        @property
+        def outputs_at_start(self) -> bool:
+            """
+            True, if the outputs are defined before the features, False otherwise.
+            """
+            attributes = self.arff_file.attributes
+            return attributes and attributes[0].name in self.output_names
+
+        @property
+        def feature_matrix(self) -> sparray:
+            """
+            The feature matrix contained in the dataset.
+            """
+            num_outputs = len(self.outputs)
+            matrix = self.arff_file.matrix
+            return matrix[:, num_outputs:] if self.outputs_at_start else matrix[:, :-num_outputs]
+
+        @property
+        def output_matrix(self) -> sparray:
+            """
+            The output matrix contained in the dataset.
+            """
+            num_outputs = len(self.outputs)
+            matrix = self.arff_file.matrix
+            return matrix[:, :num_outputs] if self.outputs_at_start else matrix[:, -num_outputs:]
+
+    @staticmethod
+    def __read_arff_file(file_path: str, dtype: np.dtype) -> ArffFile:
+        try:
+            return ArffFileSource.ArffFile.from_file(file_path, sparse=True, dtype=dtype)
+        except arff.BadLayout:
+            return ArffFileSource.ArffFile.from_file(file_path, sparse=False, dtype=dtype)
+
+    def __init__(self, directory: str):
+        """
+        :param directory: The path to the directory of the file
+        """
+        super().__init__(directory=directory, suffix=ArffFileSink.SUFFIX_ARFF)
+
+    def _read_dataset_from_file(self, state: ExperimentState, file_path: str,
+                                input_data: DatasetInputData) -> Optional[Dataset]:
+        properties = input_data.properties
+        problem_domain = state.problem_domain
+        arff_file = self.__read_arff_file(file_path=file_path, dtype=problem_domain.feature_dtype)
+        xml_file_path = path.join(path.dirname(file_path), properties.file_name + '.' + ArffFileSink.SUFFIX_XML)
+        arff_dataset = ArffFileSource.ArffDataset.from_file(arff_file=arff_file, file_path=xml_file_path)
+        return TabularDataset(x=arff_dataset.feature_matrix.tolil(),
+                              y=arff_dataset.output_matrix.astype(problem_domain.output_dtype).tolil(),
+                              features=arff_dataset.features,
+                              outputs=arff_dataset.outputs)

mlrl/testbed_arff/experiments/output/sinks/__init__.py

@@ -0,0 +1,6 @@
+"""
+Author Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow to write output data to ARFF files.
+"""
+from mlrl.testbed_arff.experiments.output.sinks.sink_arff import ArffFileSink

mlrl/testbed_arff/experiments/output/sinks/sink_arff.py

@@ -0,0 +1,97 @@
+"""
+Author Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow writing datasets to ARFF files.
+"""
+import xml.etree.ElementTree as XmlTree
+
+from xml.dom import minidom
+
+import arff
+
+from scipy.sparse import dok_array
+
+from mlrl.testbed_sklearn.experiments.dataset import AttributeType, TabularDataset
+
+from mlrl.testbed.experiments.dataset import Dataset
+from mlrl.testbed.experiments.output.sinks.sink import DatasetFileSink
+from mlrl.testbed.experiments.state import ExperimentState
+from mlrl.testbed.util.io import ENCODING_UTF8, open_writable_file
+
+from mlrl.util.options import Options
+
+
+class ArffFileSink(DatasetFileSink):
+    """
+    Allows to write a dataset to an ARFF file.
+    """
+
+    SUFFIX_ARFF = 'arff'
+
+    SUFFIX_XML = 'xml'
+
+    @staticmethod
+    def __write_arff_file(file_path: str, dataset: TabularDataset):
+        sparse = dataset.has_sparse_features and dataset.has_sparse_outputs
+        num_examples = dataset.num_examples
+        num_features = dataset.num_features
+        num_outputs = dataset.num_outputs
+
+        features = dataset.features
+        x_features = [(features[i].name, 'NUMERIC' if features[i].attribute_type == AttributeType.NUMERICAL
+                       or features[i].nominal_values is None else features[i].nominal_values)
+                      for i in range(num_features)]
+
+        outputs = dataset.outputs
+        y_features = [(outputs[i].name, 'NUMERIC' if outputs[i].attribute_type == AttributeType.NUMERICAL
+                       or outputs[i].nominal_values is None else outputs[i].nominal_values) for i in range(num_outputs)]
+
+        if sparse:
+            data = [{} for _ in range(num_examples)]
+        else:
+            data = [[0 for _ in range(num_features + num_outputs)] for _ in range(num_examples)]
+
+        for keys, value in dok_array(dataset.x).items():
+            data[keys[0]][keys[1]] = value
+
+        for keys, value in dok_array(dataset.y).items():
+            data[keys[0]][num_features + keys[1]] = value
+
+        with open_writable_file(file_path) as arff_file:
+            arff_file.write(
+                arff.dumps({
+                    'description': 'traindata',
+                    'relation': 'traindata: -C ' + str(-num_outputs),
+                    'attributes': x_features + y_features,
+                    'data': data
+                }))
+
+    @staticmethod
+    def __write_xml_file(file_path: str, dataset: TabularDataset):
+        root_element = XmlTree.Element('labels')
+        root_element.set('xmlns', 'http://mulan.sourceforge.net/labels')
+
+        for output in dataset.outputs:
+            label_element = XmlTree.SubElement(root_element, 'label')
+            label_element.set('name', output.name)
+
+        with open_writable_file(file_path) as xml_file:
+            xml_string = minidom.parseString(XmlTree.tostring(root_element)).toprettyxml(encoding=ENCODING_UTF8)
+            xml_file.write(xml_string.decode(ENCODING_UTF8))
+
+    def __init__(self, directory: str, options: Options = Options(), create_directory: bool = False):
+        """
+        :param directory:           The path to the directory, where the ARFF file should be located
+        :param options:             Options to be taken into account
+        :param create_directory:    True, if the given directory should be created, if it does not exist, False
+                                    otherwise
+        """
+        super().__init__(directory=directory,
+                         suffix=self.SUFFIX_ARFF,
+                         options=options,
+                         create_directory=create_directory)
+
+    # pylint: disable=unused-argument
+    def _write_dataset_to_file(self, file_path: str, state: ExperimentState, dataset: Dataset, **_):
+        self.__write_arff_file(file_path=file_path, dataset=dataset)
+        self.__write_xml_file(file_path=file_path.rsplit('.', 1)[0] + '.' + self.SUFFIX_XML, dataset=dataset)

mlrl_testbed_arff-0.12.0.dist-info/METADATA

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: mlrl-testbed-arff
+Version: 0.12.0
+Summary: Adds support for ARFF files to the package "mlrl-testbed"
+Author-email: Michael Rapp <michael.rapp.ml@gmail.com>
+License-Expression: MIT
+Project-URL: homepage, https://github.com/mrapp-ke/MLRL-Boomer
+Project-URL: source, https://github.com/mrapp-ke/MLRL-Boomer.git
+Project-URL: download, https://github.com/mrapp-ke/MLRL-Boomer/releases
+Project-URL: changelog, https://raw.githubusercontent.com/mrapp-ke/MLRL-Boomer/refs/heads/main/CHANGELOG.md
+Project-URL: documentation, https://mlrl-boomer.readthedocs.io/en/latest
+Project-URL: issues, https://github.com/mrapp-ke/MLRL-Boomer/issues
+Keywords: arff,machine learning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: liac-arff<2.6,>=2.5
+Requires-Dist: mlrl-testbed-sklearn==0.12.0
+
+# "MLRL-Testbed-Arff": Adds support for ARFF files to the package "mlrl-testbed"
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![PyPI version](https://badge.fury.io/py/mlrl-testbed-arff.svg)](https://badge.fury.io/py/mlrl-testbed-arff) [![Documentation Status](https://readthedocs.org/projects/mlrl-boomer/badge/?version=latest)](https://mlrl-boomer.readthedocs.io/en/latest/?badge=latest)
+
+**:link: Important links:** [API Reference](https://mlrl-boomer.readthedocs.io/en/latest/developer_guide/api/python/testbed-arff/mlrl.testbed_arff.html) | [Issue Tracker](https://github.com/mrapp-ke/MLRL-Boomer/issues) | [Changelog](https://mlrl-boomer.readthedocs.io/en/latest/misc/CHANGELOG.html) | [Contributors](https://mlrl-boomer.readthedocs.io/en/latest/misc/CONTRIBUTORS.html) | [Code of Conduct](https://mlrl-boomer.readthedocs.io/en/latest/misc/CODE_OF_CONDUCT.html) | [License](https://mlrl-boomer.readthedocs.io/en/latest/misc/LICENSE.html)
+
+This software package is an extension to the package [mlrl-testbed](https://pypi.org/project/mlrl-testbed/) which provides a command line utility for running machine learning experiments. It adds support for reading and writing tabular datasets in the [ARFF](https://waikato.github.io/weka-wiki/formats_and_processing/arff_stable/) format. For further information on the command line utility, refer to its [documentation](https://mlrl-boomer.readthedocs.io/en/latest/user_guide/testbed/index.html).
+
+## :scroll: License
+
+This project is open source software licensed under the terms of the [MIT license](https://mlrl-boomer.readthedocs.io/en/latest/misc/LICENSE.html). We welcome contributions to the project to enhance its functionality and make it more accessible to a broader audience. A frequently updated list of contributors is available [here](https://mlrl-boomer.readthedocs.io/en/latest/misc/CONTRIBUTORS.html).
+
+All contributions to the project and discussions on the [issue tracker](https://github.com/mrapp-ke/MLRL-Boomer/issues) are expected to follow the [code of conduct](https://mlrl-boomer.readthedocs.io/en/latest/misc/CODE_OF_CONDUCT.html).

mlrl_testbed_arff-0.12.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+mlrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mlrl/testbed_arff/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mlrl/testbed_arff/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mlrl/testbed_arff/experiments/input/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mlrl/testbed_arff/experiments/input/sources/__init__.py,sha256=CJvMNffltblBYgIsWzlCYfEdMxzVv52fHIOSG-SdyZ4,204
+mlrl/testbed_arff/experiments/input/sources/source_arff.py,sha256=LB9x2vuVQwFAD1OTeflBXP5Pq9U9HFD069eL3cn5TzQ,10198
+mlrl/testbed_arff/experiments/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mlrl/testbed_arff/experiments/output/sinks/__init__.py,sha256=hIL0pOCMZeCbG0yx-IV7jPbwGgfcKYqR3FziQZBqwB4,199
+mlrl/testbed_arff/experiments/output/sinks/sink_arff.py,sha256=NC7krP1srJIrMna5F2v54A2XFzTGw29r-1yB6SCeQYg,3926
+mlrl_testbed_arff-0.12.0.dist-info/METADATA,sha256=RwC81wo9WJlqLYTf5Jl27Mo7AHh5VkSIYb09R0_sE_A,3347
+mlrl_testbed_arff-0.12.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+mlrl_testbed_arff-0.12.0.dist-info/top_level.txt,sha256=o9x5YHV4Syk8twE0D3Or7SqyG69Ryp7e-HEYe8aB4j4,5
+mlrl_testbed_arff-0.12.0.dist-info/RECORD,,

mlrl_testbed_arff-0.12.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.4.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mlrl_testbed_arff-0.12.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+mlrl