smqtk_classifier 0.20.0__py3-none-any.whl

smqtk_classifier/__init__.py

@@ -0,0 +1,9 @@
+from .interfaces.classification_element import ClassificationElement  # noqa: F401
+from .interfaces.classify_descriptor import ClassifyDescriptor  # noqa: F401
+from .interfaces.classify_descriptor_supervised import ClassifyDescriptorSupervised  # noqa: F401
+from .interfaces.classify_image import ClassifyImage  # noqa: F401
+from .interfaces.classify_image_supervised import ClassifyImageSupervised  # noqa: F401
+
+from .classification_element_factory import ClassificationElementFactory  # noqa: F401
+from .classify_descriptor_collection import ClassifyDescriptorCollection  # noqa: F401
+from .classify_descriptor_collection import ClassifierCollection  # noqa: F401

smqtk_classifier/_defaults.py

@@ -0,0 +1,8 @@
+from smqtk_classifier.classification_element_factory import ClassificationElementFactory
+from smqtk_classifier.impls.classification_element.memory import MemoryClassificationElement
+
+
+# Default classifier element factory for interfaces.
+DFLT_CLASSIFIER_FACTORY = ClassificationElementFactory(
+    MemoryClassificationElement, {}
+)

smqtk_classifier/classification_element_factory.py

@@ -0,0 +1,116 @@
+from typing import Any, Dict, Hashable, Type, TypeVar
+
+from smqtk_core import Configurable
+from smqtk_core.configuration import (
+    cls_conf_from_config_dict,
+    cls_conf_to_config_dict,
+    make_default_config
+)
+from smqtk_core.dict import merge_dict
+
+from smqtk_classifier.interfaces.classification_element import ClassificationElement
+
+
+C = TypeVar("C", bound=ClassificationElement)
+
+
+class ClassificationElementFactory (Configurable):
+    """
+    Factory class for producing ClassificationElement instances of a specified
+    type and configuration.
+    """
+
+    # noinspection PyShadowingBuiltins
+    def __init__(self, type: Type[C], type_config: Dict[str, Any]):
+        """
+        Initialize the factory to produce ClassificationElement instances of the
+        given type from the given configuration.
+
+        :param type: Python implementation type of the ClassifierElement to
+            produce
+        :param type_config: Configuration dictionary that will be passed
+            ``from_config`` class method of given ``type``.
+        """
+        self.type = type
+        self.type_config = type_config
+
+    #
+    # Class methods
+    #
+
+    @classmethod
+    def get_default_config(cls) -> Dict[str, Any]:
+        """
+        Generate and return a default configuration dictionary for this class.
+        This will be primarily used for generating what the configuration
+        dictionary would look like for this class without instantiating it.
+
+        It is not be guaranteed that the configuration dictionary returned
+        from this method is valid for construction of an instance of this class.
+
+        :return: Default configuration dictionary for the class.
+        :rtype: dict
+
+        """
+        return make_default_config(ClassificationElement.get_impls())
+
+    # We likely do not expect subclasses for this type base, thus it is OK to
+    # use direct type reference in Type and return annotations.
+    @classmethod
+    def from_config(
+        cls: Type["ClassificationElementFactory"],
+        config_dict: Dict,
+        merge_default: bool = True
+    ) -> "ClassificationElementFactory":
+        """
+        Instantiate a new instance of this class given the configuration
+        JSON-compliant dictionary encapsulating initialization arguments.
+
+        This method should not be called via super unless and instance of the
+        class is desired.
+
+        :param config_dict: JSON compliant dictionary encapsulating
+            a configuration.
+        :param merge_default: Merge the given configuration on top of the
+            default provided by ``get_default_config``.
+
+        :return: Constructed instance from the provided config.
+        """
+        if merge_default:
+            config_dict = merge_dict(cls.get_default_config(), config_dict)
+
+        ce_type, ce_conf = cls_conf_from_config_dict(
+            config_dict,  ClassificationElement.get_impls()
+        )
+        return ClassificationElementFactory(ce_type, ce_conf)
+
+    def get_config(self) -> Dict[str, Any]:
+        return cls_conf_to_config_dict(self.type, self.type_config)
+
+    # noinspection PyShadowingBuiltins
+    def new_classification(self, type: str, uuid: Hashable) -> ClassificationElement:
+        """
+        Create a new ClassificationElement instance of the configured
+        implementation.
+
+        :param type: Type of classifier. This is usually the name of the
+            classifier that generated this result.
+        :param uuid: UUID to associate with the classification.
+
+        :return: New ClassificationElement instance.
+        """
+        return self.type.from_config(self.type_config, type, uuid)
+
+    # noinspection PyShadowingBuiltins
+    def __call__(self, type: str, uuid: Hashable) -> ClassificationElement:
+        """
+        Create a new ClassificationElement instance of the configured
+        implementation.
+
+        :param type: Type of classifier. This is usually the name of the
+            classifier that generated this result.
+        :param uuid: UUID to associate with the classification.
+
+        :return: New ClassificationElement instance.
+        """
+        return self.new_classification(type, uuid)

smqtk_classifier/classify_descriptor_collection.py

@@ -0,0 +1,298 @@
+import threading
+from typing import Any, Dict, Hashable, Iterable, List, Optional, Mapping, Sequence, Set, Type
+from types import TracebackType
+from warnings import warn
+import numpy as np
+
+from smqtk_core.configuration import (
+    Configurable,
+    make_default_config,
+    from_config_dict,
+    to_config_dict
+)
+from smqtk_core.dict import merge_dict
+from smqtk_descriptors import DescriptorElement
+
+from smqtk_classifier.classification_element_factory import ClassificationElementFactory
+from smqtk_classifier.exceptions import MissingLabelError
+from smqtk_classifier.interfaces.classification_element import ClassificationElement
+
+from ._defaults import DFLT_CLASSIFIER_FACTORY
+from .interfaces.classify_descriptor import ClassifyDescriptor
+
+
+class ClassifyDescriptorCollection (Configurable):
+    """
+    A collection of descriptively-labeled classifier instances for the purpose
+    of applying all stored classifiers to one or more input descriptor
+    elements.
+
+    TODO: [optionally?] map a classification element factory per classifier.
+
+    :param classifiers: Optional dictionary of semantic label keys and
+        Classifier instance values.
+    :param labeled_classifiers: Key-word arguments may be provided where
+        the key used is considered the semantic label of the provided
+        Classifier instance.
+    """
+
+    EXAMPLE_KEY = '__example_label__'
+
+    def __init__(
+            self,
+            classifiers: Mapping[str, ClassifyDescriptor] | None = None,
+            **labeled_classifiers: ClassifyDescriptor
+    ):
+        self._label_to_classifier_lock = threading.RLock()
+        self._label_to_classifier = {}
+
+        # Go though classifiers map and key-word arguments, check that values
+        # are actually classifiers.
+        if classifiers is not None:
+            for label, classifier in classifiers.items():
+                if not isinstance(classifier, ClassifyDescriptor):
+                    raise ValueError("Found a non-Classifier instance value "
+                                     "for key '%s'" % label)
+                self._label_to_classifier[label] = classifier
+
+        for label, classifier in labeled_classifiers.items():
+            if not isinstance(classifier, ClassifyDescriptor):
+                raise ValueError("Found a non-Classifier instance value "
+                                 "for key '%s'" % label)
+            elif label in self._label_to_classifier:
+                raise ValueError("Duplicate classifier label '%s' provided "
+                                 "in key-word arguments." % label)
+            self._label_to_classifier[label] = classifier
+
+    @classmethod
+    def get_default_config(cls) -> Dict[str, Any]:
+        c = super(ClassifyDescriptorCollection, cls).get_default_config()
+
+        # We list the label-classifier mapping on one level, so remove the
+        # nested map parameter that can optionally be used in the constructor.
+        del c['classifiers']
+
+        # Add slot of a list of classifier plugin specifications
+        c[cls.EXAMPLE_KEY] = make_default_config(ClassifyDescriptor.get_impls())
+
+        return c
+
+    # We likely do not expect subclasses for this type base, thus it is OK to
+    # use direct type reference in Type and return annotations.
+    @classmethod
+    def from_config(
+        cls,
+        config_dict: Dict,
+        merge_default: bool = True
+    ) -> "ClassifyDescriptorCollection":
+        if merge_default:
+            config_dict = merge_dict(cls.get_default_config(), config_dict)
+
+        classifier_map = {}
+
+        # Copying list of keys so we can update the dictionary as we loop.
+        for label in list(config_dict.keys()):
+            # Skip the example section.
+            if label == cls.EXAMPLE_KEY:
+                continue
+
+            classifier_config = config_dict[label]
+            classifier = from_config_dict(classifier_config,
+                                          ClassifyDescriptor.get_impls())
+            classifier_map[label] = classifier
+
+        return cls(classifiers=classifier_map)
+
+    def get_config(self) -> Dict[str, Any]:
+        with self._label_to_classifier_lock:
+            c = dict((label, to_config_dict(classifier))
+                     for label, classifier
+                     in self._label_to_classifier.items())
+        return c
+
+    def __enter__(self) -> "ClassifyDescriptorCollection":
+        """
+        :rtype: IqrSession
+        """
+        self._label_to_classifier_lock.acquire()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType]
+    ) -> None:
+        self._label_to_classifier_lock.release()
+
+    def size(self) -> int:
+        with self._label_to_classifier_lock:
+            return len(self._label_to_classifier)
+
+    __len__ = size
+
+    def labels(self) -> Set[str]:
+        """
+        :return: Set of labels for currently collected classifiers.
+        """
+        with self._label_to_classifier_lock:
+            return set(self._label_to_classifier.keys())
+
+    def add_classifier(self, label: str, classifier: ClassifyDescriptor) -> "ClassifyDescriptorCollection":
+        """
+        Add a classifier instance with associated descriptive label to this
+        collection.
+
+        :param label: String descriptive label for the classifier.
+        :param classifier: Classifier instance to collect.
+
+        :raises ValueError: Classifier provided is not actually a classifier
+            instance, or if the label provided already exists in this
+            collection.
+
+        :return: Self.
+        """
+        if not isinstance(classifier, ClassifyDescriptor):
+            raise ValueError("Not given a Classifier instance (given type"
+                             " %s)." % type(classifier))
+        with self._label_to_classifier_lock:
+            if label in self._label_to_classifier:
+                raise ValueError("Duplicate label provided: '%s'" % label)
+            self._label_to_classifier[label] = classifier
+        return self
+
+    def get_classifier(self, label: str) -> ClassifyDescriptor:
+        """
+        Get the classifier instance for a given label.
+
+        :param label: Label of the classifier to get.
+
+        :raises KeyError: No classifier for the given label.
+
+        :return: Classifier instance.
+        """
+        with self._label_to_classifier_lock:
+            return self._label_to_classifier[label]
+
+    def remove_classifier(self, label: str) -> "ClassifyDescriptorCollection":
+        """
+        Remove a label-classifier pair from this collection.
+
+        :param label: Label of the classifier to remove.
+
+        :raises KeyError: The given label does not reference a classifier in
+            this collection.
+
+        :return: Self.
+        """
+        with self._label_to_classifier_lock:
+            del self._label_to_classifier[label]
+        return self
+
+    def labels_to_classifiers(self, labels: Optional[Iterable[str]] = None) -> Dict[str, ClassifyDescriptor]:
+        """
+        Get a shallow copy mapping of classifiers for the labels given, or for
+        all classifiers if no labels were explicitly given.
+
+        This method is thread-safe and the returned dictionary is separate from
+        this class's control. However, the classifier instances are still
+        shared.
+
+        :param None | typing.Iterable[str] labels:
+            One or more labels of stored classifiers to retrieve.
+            If None, we will consider all stored classifiers.
+
+        :raises MissingLabelError: Thrown when one or more labels provided do
+            not associate to any currently stored classifiers.
+
+        :return: Dictionary mapping string labels to Classifier instances.
+        """
+        with self._label_to_classifier_lock:
+            if labels is not None:
+                labels = list(labels)
+                # If we're missing some of the requested labels, complain
+                missing_labels = set(labels) - self.labels()
+                if missing_labels:
+                    raise MissingLabelError(missing_labels)
+                label2classifier = {label: self._label_to_classifier[label]
+                                    for label in labels}
+            else:
+                label2classifier = dict(self._label_to_classifier)
+        return label2classifier
+
+    def classify(
+        self,
+        descriptor: DescriptorElement,
+        labels: Optional[Iterable[str]] = None,
+        factory: ClassificationElementFactory = DFLT_CLASSIFIER_FACTORY,
+        overwrite: bool = False
+    ) -> Dict[str, ClassificationElement]:
+        """
+        Apply all stored classifiers to the given descriptor element.
+
+        We return a dictionary mapping the label of a stored classifier to the
+        classifier element result produced by that classifier via the
+        provided classification element factory.
+
+        :param descriptor: Descriptor element to classify.
+        :param labels: One or more labels of stored classifiers to use for
+            classifying the given descriptor.  If None, use all stored
+            classifiers.
+        :param factory: Classification element factory.
+        :param overwrite: Force re-computation of the classification of the
+            input descriptor.
+
+        :raises smqtk.exceptions.MissingLabelError: Some or all of the
+            requested labels are missing.
+
+        :return: Result dictionary of classifier labels to classification
+            elements.
+        """
+        d_classifications = {}
+        label2classifier = self.labels_to_classifiers(labels)
+        for label, classifier in label2classifier.items():
+            d_classifications[label] = classifier.classify_one_element(
+                descriptor, factory=factory, overwrite=overwrite
+            )
+        return d_classifications
+
+    def classify_arrays(
+        self,
+        array_seq: Sequence[np.ndarray],
+        labels: Sequence[str] | None = None
+    ) -> Dict[str, List[Dict[Hashable, float]]]:
+        """
+        Apply all stored classifiers to the given iterable or matrix of numpy
+        arrays.
+
+        We return a dictionary mapping the label of a stored classifier to the
+        class-confidence map result produced by that classifier's
+        `classify_arrays` method.
+
+        :param Sequence[numpy.ndarray] array_seq:
+            Sequence of descriptor vectors, as numpy arrays, to be classified.
+        :param Sequence[str] labels:
+            One or more labels of stored classifiers to use for classifying the
+            given descriptors.  If None (the default), use all stored
+            classifiers.
+
+        :raises MissingLabelError: Thrown when one or more labels provided do
+            not associate to any currently stored classifiers.
+
+        :return: Dictionary of result predictions for each classifier.
+        :rtype: dict[str, list[dict[collections.abc.Hashable, float]]]
+        """
+        label2classifier = self.labels_to_classifiers(labels)
+        label2pred = {}
+        for label, classifier in label2classifier.items():
+            label2pred[label] = list(classifier.classify_arrays(array_seq))
+        return label2pred
+
+
+class ClassifierCollection(ClassifyDescriptorCollection):
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        warn("ClassifierCollection was renamed to "
+             "ClassifyDescriptorCollection", category=DeprecationWarning,
+             stacklevel=2)
+        super().__init__(*args, **kwargs)

smqtk_classifier/exceptions.py

@@ -0,0 +1,29 @@
+from typing import Set
+
+
+class NoClassificationError (Exception):
+    """
+    When a ClassificationElement has no mapping yet set, but an operation
+    required it.
+    """
+
+
+class MissingLabelError(Exception):
+    """
+    Raised by ClassifyDescriptorCollection.classify when requested classifier
+    labels are missing from collection.
+    """
+    def __init__(self, labels: Set[str]):
+        """
+        :param labels: The labels missing from the collection
+        """
+        super(MissingLabelError, self).__init__(labels)
+        self.labels = labels
+
+
+class ExistingModelError(Exception):
+    """
+    Raised by ClassifyDescriptorSupervised and ClassifyImageSupervised when
+    a model already exists in an instance to prevent overwriting and existing
+    model
+    """

smqtk_classifier/impls/classification_element/file.py

@@ -0,0 +1,125 @@
+import pickle
+import os.path as osp
+from typing import Any, Dict, Hashable, Optional
+
+from smqtk_dataprovider.utils.file import safe_create_dir
+from smqtk_dataprovider.utils.string import partition_string
+
+from smqtk_classifier.exceptions import NoClassificationError
+from smqtk_classifier.interfaces.classification_element import (
+    ClassificationElement,
+    CLASSIFICATION_DICT_T,
+    CLASSIFICATION_MAP_T
+)
+
+
+class FileClassificationElement (ClassificationElement):  # lgtm [py/missing-equals]
+    """
+    Create a classification element from a cached file representation.
+
+    :param type_name: Type of classification. This is usually the name of
+        the classifier that generated this result.
+    :type type_name: str
+
+    :param uuid: uuid for this classification
+    :type uuid: collections.abc.Hashable
+
+    :param save_dir: Directory to save this element's contents. If this path
+        is relative, we interpret as relative to the current working
+        directory.
+    :type save_dir: str | unicode
+
+    :param subdir_split: If a positive integer, this will cause us to store
+        the vector file in a subdirectory under the ``save_dir`` that was
+        specified. The integer value specifies the number of splits that we
+        will make in the stringification of this descriptor's UUID. If
+        there happen to be dashes in this stringification, we will
+        remove them (as would happen if given an uuid.UUID instance as
+        the uuid element).
+    :type subdir_split: None | int
+
+    :param pickle_protocol: Pickling protocol to use. We will use -1 by
+        default (latest version, probably binary).
+    :type pickle_protocol: int
+    """
+
+    __slots__ = ('save_dir', 'pickle_protocol', 'subdir_split', 'filepath')
+
+    def __init__(
+        self,
+        type_name: str,
+        uuid: Hashable,
+        save_dir: str,
+        subdir_split: Optional[int] = None,
+        pickle_protocol: int = -1
+    ):
+        super(FileClassificationElement, self).__init__(type_name, uuid)
+
+        # TODO: Remove absolute path conversion (allow relative)
+        self.save_dir = osp.abspath(osp.expanduser(save_dir))
+        self.pickle_protocol = pickle_protocol
+
+        # Saving components
+        self.subdir_split = subdir_split
+        if subdir_split and int(subdir_split) > 0:
+            self.subdir_split = subdir_split = int(subdir_split)
+            # Using all but the last split segment. This is so we don't create
+            # a whole bunch of directories with a single element in them.
+            save_dir = osp.join(self.save_dir,
+                                *partition_string(str(uuid).replace('-', ''),
+                                                  subdir_split)[:subdir_split-1]
+                                )
+        else:
+            save_dir = self.save_dir
+
+        self.filepath = osp.join(save_dir,
+                                 "%s.%s.classification.pickle"
+                                 % (self.type_name, str(self.uuid)))
+
+    @classmethod
+    def is_usable(cls) -> bool:
+        return True
+
+    def get_config(self) -> Dict[str, Any]:
+        return {
+            "save_dir": self.save_dir,
+            'subdir_split': self.subdir_split,
+            "pickle_protocol": self.pickle_protocol,
+        }
+
+    def __getstate__(self) -> Any:
+        return (
+            super(FileClassificationElement, self).__getstate__(),
+            self.save_dir,
+            self.pickle_protocol,
+            self.subdir_split,
+            self.filepath,
+        )
+
+    def __setstate__(self, state: Any) -> None:
+        super(FileClassificationElement, self).__setstate__(state[0])
+        self.save_dir, \
+            self.pickle_protocol, \
+            self.subdir_split, \
+            self.filepath \
+            = state[1:]
+
+    def has_classifications(self) -> bool:
+        return osp.isfile(self.filepath)
+
+    def get_classification(self) -> CLASSIFICATION_DICT_T:
+        if not self.has_classifications():
+            raise NoClassificationError("No classification values.")
+        with open(self.filepath, 'rb') as f:
+            return pickle.load(f)
+
+    def set_classification(
+        self,
+        m: Optional[CLASSIFICATION_MAP_T] = None,
+        **kwds: float
+    ) -> CLASSIFICATION_DICT_T:
+        m = super(FileClassificationElement, self).set_classification(m, **kwds)
+        safe_create_dir(osp.dirname(self.filepath))
+        with open(self.filepath, 'wb') as f:
+            pickle.dump(m, f, self.pickle_protocol)
+        return m

smqtk_classifier/impls/classification_element/memory.py

@@ -0,0 +1,77 @@
+from threading import RLock
+from typing import Any, Dict, Hashable, Optional
+
+from smqtk_classifier.exceptions import NoClassificationError
+from smqtk_classifier.interfaces.classification_element import (
+    ClassificationElement, CLASSIFICATION_MAP_T, CLASSIFICATION_DICT_T
+)
+
+
+class MemoryClassificationElement (ClassificationElement):  # lgtm [py/missing-equals]
+    """
+    In-memory representation of classification results. This is represented
+    with a python dictionary.
+
+    :param type_name: Name of the type of classifier this classification
+        was generated by.
+    :type type_name: str
+
+    :param uuid: Unique ID reference of the classification
+    :type uuid: collections.abc.Hashable
+    """
+
+    __slots__ = ('_c', '_c_lock')
+
+    def __init__(self, type_name: str, uuid: Hashable):
+        super(MemoryClassificationElement, self).__init__(type_name, uuid)
+
+        # dictionary of classification labels and values
+        self._c: Optional[CLASSIFICATION_DICT_T] = None
+        # Cannot be pickled. New lock initialized upon pickle/unpickle
+        self._c_lock = RLock()
+
+    @classmethod
+    def is_usable(cls) -> bool:
+        # No external dependencies
+        return True
+
+    def get_config(self) -> Dict[str, Any]:
+        return {}
+
+    def __getstate__(self) -> Any:
+        state = {
+            'parent': super(MemoryClassificationElement, self).__getstate__(),
+        }
+        with self._c_lock:
+            state['c'] = self._c
+        return state
+
+    def __setstate__(self, state: Any) -> None:
+        super(MemoryClassificationElement, self).__setstate__(state['parent'])
+        if not hasattr(self, '_c_lock') or self._c_lock is None:
+            self._c_lock = RLock()
+        with self._c_lock:
+            #: :type: None | dict[collections.abc.Hashable, float]
+            self._c = state['c']
+
+    def has_classifications(self) -> bool:
+        with self._c_lock:
+            return bool(self._c)
+
+    def get_classification(self) -> CLASSIFICATION_DICT_T:
+        with self._c_lock:
+            if self._c:
+                return self._c
+            else:
+                raise NoClassificationError("No classification labels/values")
+
+    def set_classification(
+        self,
+        m: Optional[CLASSIFICATION_MAP_T] = None,
+        **kwds: float
+    ) -> CLASSIFICATION_DICT_T:
+        m = super(MemoryClassificationElement, self)\
+            .set_classification(m, **kwds)
+        with self._c_lock:
+            self._c = m
+        return m