genelastic 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

genelastic/common/elastic.py

@@ -1,4 +1,3 @@
-# pylint: disable=missing-module-docstring
 import datetime
 import logging
 import time
@@ -13,14 +12,15 @@ from elasticsearch import Elasticsearch
 from .exceptions import DBIntegrityError
 from .types import Bucket, BulkItems
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
-class ElasticConn(ABC):  # pylint: disable=too-few-public-methods
+class ElasticConn(ABC):
     """Abstract class representing a connector for an Elasticsearch server."""
+
     client: Elasticsearch
 
-    def __init__(self, url: str, fingerprint: str, **kwargs: Any):
+    def __init__(self, url: str, fingerprint: str, **kwargs: Any) -> None:  # noqa: ANN401
         """Initialize an elasticsearch client instance.
 
         :url: URL of the Elasticsearch host.
@@ -34,41 +34,49 @@ class ElasticConn(ABC):  # pylint: disable=too-few-public-methods
                 ssl_assert_fingerprint=fingerprint,
                 # Verify cert only when the fingerprint is not None.
                 verify_certs=bool(fingerprint),
-                **kwargs
+                **kwargs,
             )
             self.client.info()
-        except (elastic_transport.TransportError, elasticsearch.AuthenticationException) as e:
+        except (
+            elastic_transport.TransportError,
+            elasticsearch.AuthenticationException,
+        ) as e:
             raise SystemExit(e) from e
 
 
-class ElasticImportConn(ElasticConn):  # pylint: disable=too-few-public-methods
+class ElasticImportConn(ElasticConn):
     """Connector to import data into an Elasticsearch database."""
-    def import_items(self, bulk_items: BulkItems,
-                     start_time: float,
-                     total_items: int) -> None:
+
+    def import_items(
+        self, bulk_items: BulkItems, start_time: float, total_items: int
+    ) -> None:
         """Import items to the Elasticsearch database."""
         if len(bulk_items) > 0:
             elasticsearch.helpers.bulk(self.client, bulk_items)
         elapsed = time.perf_counter() - start_time
-        logger.info("Imported %d items in %s (%f items/s).", total_items,
-                    datetime.timedelta(seconds=elapsed), total_items / elapsed)
+        logger.info(
+            "Imported %d items in %s (%f items/s).",
+            total_items,
+            datetime.timedelta(seconds=elapsed),
+            total_items / elapsed,
+        )
 
 
 class ElasticQueryConn(ElasticConn):
     """Connector to query data from an Elasticsearch database."""
 
-    def get_indices(self) -> Any | str:
+    def get_indices(self) -> Any | str:  # noqa: ANN401
         """Return all indices."""
         return self.client.cat.indices(format="json").body
 
-    def get_document_by_id(self, index: str, document_id: str) -> Any | str:
+    def get_document_by_id(self, index: str, document_id: str) -> Any | str:  # noqa: ANN401
         """Return a document by its ID."""
         return self.client.get(index=index, id=document_id).body
 
-    def run_composite_aggregation(self, index: str, query: dict[str, typing.Any]) \
-            -> list[Bucket]:
-        """
-        Executes a composite aggregation on an Elasticsearch index and
+    def run_composite_aggregation(
+        self, index: str, query: dict[str, typing.Any]
+    ) -> list[Bucket]:
+        """Executes a composite aggregation on an Elasticsearch index and
         returns all paginated results.
 
         :param index: Name of the index to query.
@@ -77,29 +85,39 @@ class ElasticQueryConn(ElasticConn):
         """
         # Extract the aggregation name from the query dict.
         agg_name = next(iter(query["aggs"]))
-        all_buckets: typing.List[Bucket] = []
+        all_buckets: list[Bucket] = []
 
         try:
-            logger.debug("Running composite aggregation query %s on index '%s'.", query, index)
+            logger.debug(
+                "Running composite aggregation query %s on index '%s'.",
+                query,
+                index,
+            )
             response = self.client.search(index=index, body=query)
         except elasticsearch.NotFoundError as e:
-            raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
+            msg = f"Error: {e.message} for index '{index}'."
+            raise SystemExit(msg) from e
 
         while True:
             # Extract buckets from the response.
-            buckets: typing.List[Bucket] = response['aggregations'][agg_name]['buckets']
+            buckets: list[Bucket] = response["aggregations"][agg_name][
+                "buckets"
+            ]
             all_buckets.extend(buckets)
 
             # Check if there are more results to fetch.
-            if 'after_key' in response['aggregations'][agg_name]:
-                after_key = response['aggregations'][agg_name]['after_key']
-                query['aggs'][agg_name]['composite']['after'] = after_key
+            if "after_key" in response["aggregations"][agg_name]:
+                after_key = response["aggregations"][agg_name]["after_key"]
+                query["aggs"][agg_name]["composite"]["after"] = after_key
                 try:
-                    logger.debug("Running query %s on index '%s'.", query, index)
+                    logger.debug(
+                        "Running query %s on index '%s'.", query, index
+                    )
                     # Fetch the next page of results.
                     response = self.client.search(index=index, body=query)
                 except elasticsearch.NotFoundError as e:
-                    raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
+                    msg = f"Error: {e.message} for index '{index}'."
+                    raise SystemExit(msg) from e
             else:
                 break
 
@@ -114,25 +132,34 @@ class ElasticQueryConn(ElasticConn):
             "aggs": {
                 "get_field_values": {
                     "composite": {
-                        "sources": {"values": {"terms": {"field": f"{field_name}.keyword"}}},
+                        "sources": {
+                            "values": {
+                                "terms": {"field": f"{field_name}.keyword"}
+                            }
+                        },
                         "size": 1000,
                     }
                 }
-            }
+            },
         }
 
-        buckets: typing.List[Bucket] = self.run_composite_aggregation(index, query)
+        buckets: list[Bucket] = self.run_composite_aggregation(index, query)
 
         for bucket in buckets:
-            values.add(bucket['key']['values'])
+            values.add(bucket["key"]["values"])
 
         return values
 
-    def search_by_field_value(self, index: str, field: str, value: str) -> (
-            typing.Dict[str, typing.Any] | None):
+    def search_by_field_value(
+        self, index: str, field: str, value: str
+    ) -> dict[str, typing.Any] | None:
         """Search a document by a value for a certain field."""
-        logger.info("Searching for field '%s' with value '%s' inside index '%s'.",
-                    field, value, index)
+        logger.info(
+            "Searching for field '%s' with value '%s' inside index '%s'.",
+            field,
+            value,
+            index,
+        )
         search_query = {
             "query": {
                 "term": {
@@ -144,22 +171,23 @@ class ElasticQueryConn(ElasticConn):
         response = self.client.search(index=index, body=search_query)
 
         try:
-            return response['hits']['hits'][0]['_source']  # type: ignore
+            return response["hits"]["hits"][0]["_source"]  # type: ignore[no-any-return]
         except KeyError:
             return None
 
     def ensure_unique(self, index: str, field: str) -> None:
-        """
-        Ensure that all values of a field in an index are all unique.
+        """Ensure that all values of a field in an index are all unique.
 
         :param index: Name of the index.
         :param field: Field name to check for value uniqueness.
         :raises genelastic.common.DBIntegrityError:
             Some values of the given field are duplicated in the index.
         """
-
-        logger.info("Ensuring that the field '%s' in the index '%s' only contains unique values...",
-                    field, index)
+        logger.info(
+            "Ensuring that the field '%s' in the index '%s' only contains unique values...",
+            field,
+            index,
+        )
         query = {
             "size": 0,
             "aggs": {
@@ -167,17 +195,20 @@ class ElasticQueryConn(ElasticConn):
                     "terms": {
                         "field": f"{field}.keyword",
                         "size": 10000,
-                        "min_doc_count": 2
+                        "min_doc_count": 2,
                     }
                 }
-            }
+            },
+        }
+        buckets: list[Bucket] = self.run_composite_aggregation(index, query)
+        duplicated_processes: set[str] = {
+            str(bucket["key"]) for bucket in buckets
         }
-        buckets: typing.List[Bucket] = self.run_composite_aggregation(index, query)
-        duplicated_processes: typing.Set[str] = set(map(lambda bucket: str(bucket["key"]), buckets))
 
         if len(duplicated_processes) > 0:
-            raise DBIntegrityError(f"Found non-unique value for field {field} in index '{index}': "
-                                   f"{', '.join(duplicated_processes)}.")
+            msg = f"Found non-unique value for field {field} in index '{index}': {', '.join(duplicated_processes)}."
+            raise DBIntegrityError(msg)
 
-        logger.info("All values of field '%s' in index '%s' are unique.",
-                    field, index)
+        logger.info(
+            "All values of field '%s' in index '%s' are unique.", field, index
+        )

genelastic/common/exceptions.py

@@ -1,5 +1,3 @@
-# pylint: disable=missing-module-docstring
-
 class DBIntegrityError(Exception):
     """Represents an integrity error,
     raised when the database content does not match the expected data schema.

genelastic/common/types.py

@@ -1,20 +1,25 @@
-# pylint: disable=missing-module-docstring
-
 import typing
 
+# Types related to Elasticsearch data import.
 Bucket: typing.TypeAlias = dict[str, dict[typing.Any, typing.Any]]
+BundleDict: typing.TypeAlias = dict[str, typing.Any]
+
+AnalysisMetaData: typing.TypeAlias = dict[str, str | int]
+WetProcessesData: typing.TypeAlias = dict[str, str | int | float]
+BioInfoProcessData: typing.TypeAlias = dict[str, str | list[str]]
 
-AnalysisMetaData: typing.TypeAlias = typing.Dict[str, str | int]
-WetProcessesData: typing.TypeAlias = typing.Dict[str, str | int | float]
-BioInfoProcessData: typing.TypeAlias = typing.Dict[str, str | typing.List[str]]
-BundleDict: typing.TypeAlias = typing.Dict[str, typing.Any]
+AnalysisDocument: typing.TypeAlias = dict[str, str | None | AnalysisMetaData]
+MetadataDocument: typing.TypeAlias = dict[
+    str, int | str | list[typing.Any | None]
+]
+ProcessDocument: typing.TypeAlias = (
+    dict[str, str] | WetProcessesData | BioInfoProcessData
+)
+BulkItems: typing.TypeAlias = list[
+    dict[str, str | MetadataDocument | AnalysisDocument | ProcessDocument]
+]
 
-AnalysisDocument: typing.TypeAlias = typing.Dict[str, str | None | AnalysisMetaData]
-MetadataDocument: typing.TypeAlias = typing.Dict[str, int | str | typing.List[typing.Any | None]]
-ProcessDocument: typing.TypeAlias = (typing.Dict[str, str] |
-                                     WetProcessesData |
-                                     BioInfoProcessData)
-BulkItems: typing.TypeAlias = typing.List[typing.Dict[str, str |
-                                                           MetadataDocument |
-                                                           AnalysisDocument |
-                                                           ProcessDocument]]
+# Types related to random bundle generation.
+RandomBiProcessData: typing.TypeAlias = dict[str, str | list[dict[str, str]]]
+RandomWetProcessData: typing.TypeAlias = dict[str, str | float]
+RandomAnalysisData: typing.TypeAlias = dict[str, str | list[int | str]]

genelastic/import_data/__init__.py

@@ -1,9 +1,27 @@
 """Genelastic package for importing Genomic data into Elasticsearch."""
+
 from .analysis import Analysis
-from .import_bundle_factory import (make_import_bundle_from_files,
-                                    load_import_bundle_file)
-from .tags import Tags
 from .import_bundle import ImportBundle
+from .import_bundle_factory import (
+    load_import_bundle_file,
+    make_import_bundle_from_files,
+)
+from .random_bundle import (
+    RandomAnalysis,
+    RandomBiProcess,
+    RandomBundle,
+    RandomWetProcess,
+)
+from .tags import Tags
 
-__all__ = ['Analysis', 'Tags', 'ImportBundle', 'make_import_bundle_from_files',
-           'load_import_bundle_file']
+__all__ = [
+    "Analysis",
+    "ImportBundle",
+    "RandomAnalysis",
+    "RandomBiProcess",
+    "RandomBundle",
+    "RandomWetProcess",
+    "Tags",
+    "load_import_bundle_file",
+    "make_import_bundle_from_files",
+]

genelastic/import_data/analyses.py

@@ -1,23 +1,22 @@
-# pylint: disable=missing-module-docstring
 import typing
 
-from  genelastic.common import BundleDict
+from genelastic.common import BundleDict
 
 from .analysis import Analysis
 from .data_file import DataFile
 
+
 class Analyses:
-    """Class Analyses is a container of Analysis objects.
-    """
+    """Class Analyses is a container of Analysis objects."""
 
     def __init__(self) -> None:
-        self._arr: typing.List[Analysis] = []
+        self._arr: list[Analysis] = []
         self._iter_index: int = 0
 
     def __len__(self) -> int:
         return len(self._arr)
 
-    def __iter__(self) -> typing.Generator[Analysis, typing.Any, None]:
+    def __iter__(self) -> typing.Iterator[Analysis]:
         yield from self._arr
 
     @typing.overload
@@ -25,13 +24,13 @@ class Analyses:
         pass
 
     @typing.overload
-    def __getitem__(self, k: slice) -> typing.List[Analysis]:
+    def __getitem__(self, k: slice) -> list[Analysis]:
         pass
 
-    def __getitem__(self, k): # type: ignore
+    def __getitem__(self, k):  # type: ignore[no-untyped-def]
         if isinstance(k, int):
             return self._arr[k]
-        return self._arr[k.start:k.stop]
+        return self._arr[k.start : k.stop]
 
     def add(self, a: Analysis) -> None:
         """Add one Analysis object."""
@@ -39,20 +38,18 @@ class Analyses:
 
     def get_nb_files(self, cat: str | None = None) -> int:
         """Get the total number of files as paths."""
-        return len(self.get_data_files(cat = cat))
-
-    def get_data_files(self, cat: str | None = None) -> typing.List[DataFile]:
-        """Get the total number of files as DataFile objects.
-        """
+        return len(self.get_data_files(cat=cat))
 
-        data_files: typing.List[DataFile] = []
+    def get_data_files(self, cat: str | None = None) -> list[DataFile]:
+        """Get the total number of files as DataFile objects."""
+        data_files: list[DataFile] = []
 
         for a in self._arr:
-            data_files.extend(a.get_data_files(cat = cat))
+            data_files.extend(a.get_data_files(cat=cat))
 
         return data_files
 
-    def get_all_categories(self) -> typing.Set[str]:
+    def get_all_categories(self) -> set[str]:
         """Return all the categories of the analyses."""
         categories = set()
         for a in self._arr:
@@ -60,10 +57,10 @@ class Analyses:
         return categories
 
     @classmethod
-    def from_array_of_dicts(cls, arr: typing.Sequence[BundleDict]
-                            ) -> typing.Self:
+    def from_array_of_dicts(
+        cls, arr: typing.Sequence[BundleDict]
+    ) -> typing.Self:
         """Build an Analyses instance."""
-
         analyses = cls()
 
         for d in arr:

genelastic/import_data/analysis.py

@@ -1,8 +1,5 @@
-# pylint: disable=missing-module-docstring
 import copy
-import glob
 import logging
-import os
 import re
 import typing
 from pathlib import Path
@@ -14,28 +11,31 @@ from .data_file import DataFile
 from .filename_pattern import FilenamePattern
 from .tags import Tags
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
 class Analysis:
     """Class Analysis that represents an analysis."""
 
-    # pylint: disable-next=too-many-arguments, too-many-positional-arguments
-    def __init__(self,
-                 tags: Tags,
-                 root_dir: str = '.',
-                 bundle_file: str | None = None,
-                 file_prefix: str | None = None,
-                 files: typing.Sequence[str] | None = None,
-                 data_path: str | None = None,
-                 **metadata: str | int) -> None:
-        self._bundle_file = bundle_file
+    def __init__(  # noqa: PLR0913
+        self,
+        tags: Tags,
+        root_dir: str = ".",
+        bundle_file: str | None = None,
+        file_prefix: str | None = None,
+        files: typing.Sequence[str] | None = None,
+        data_path: str | None = None,
+        **metadata: str | int,
+    ) -> None:
+        self._bundle_file = Path(bundle_file) if bundle_file else None
         self._file_prefix = file_prefix
         self._files = files
-        self._data_path = Analysis._resolve_data_path(root_dir, data_path)
+        self._data_path = Analysis._resolve_data_path(
+            Path(root_dir), Path(data_path) if data_path else None
+        )
         self._tags = tags
         self._metadata: AnalysisMetaData = metadata
-        self._categories: typing.Set[str] = set()
+        self._categories: set[str] = set()
 
     @property
     def metadata(self) -> AnalysisMetaData:
@@ -43,17 +43,15 @@ class Analysis:
         return copy.deepcopy(self._metadata)
 
     @property
-    def bundle_file(self) -> str | None:
+    def bundle_file(self) -> Path | None:
         """Get the bundle file."""
         return self._bundle_file
 
     @property
     def filename_regex(self) -> str:
-        """
-        Resolve placeholders in a file prefix using metadata
+        """Resolve placeholders in a file prefix using metadata
         and unresolved placeholders are converted to regex groups
         """
-
         x: str = r"^.+\.(?P<ext>vcf|cov)(\.gz)?$"
 
         # Use existing generic prefix
@@ -65,84 +63,87 @@ class Analysis:
                 regex = tag_attrs["regex"]
 
                 # Build field regex
-                field_regex = (f"(?P<{field}>{self._metadata.get(field)})"
-                               if field in self._metadata else
-                               f"(?P<{field}>{regex})")
+                field_regex = (
+                    f"(?P<{field}>{self._metadata.get(field)})"
+                    if field in self._metadata
+                    else f"(?P<{field}>{regex})"
+                )
                 # Replace tag with field regex
                 x = x.replace(tag_name, field_regex)
 
             # Check for tags that were not replaced.
             groups = re.findall(self._tags.search_regex, x)
             for match in groups:
-                logger.warning("String '%s' in key 'file_prefix' looks like an undefined tag. "
-                               "If this string is not a tag, you can ignore this warning.",
-                               match)
+                logger.warning(
+                    "String '%s' in key 'file_prefix' looks like an undefined tag. "
+                    "If this string is not a tag, you can ignore this warning.",
+                    match,
+                )
 
             # Add missing start and end markers
             if not x.startswith("^"):
                 x = "^" + x
             if not x.endswith("$"):
-                x += (r"\.(?P<ext>" + '|'.join(ALLOWED_CATEGORIES)
-                      + r")(\.gz)?$")
+                x += r"\.(?P<ext>" + "|".join(ALLOWED_CATEGORIES) + r")(\.gz)?$"
             logger.debug("File regex for %s: %s", self._bundle_file, x)
 
         return x
 
     def get_nb_files(self, cat: str | None = None) -> int:
-        """Returns the total number of files.
-        """
+        """Returns the total number of files."""
         return len(self.get_file_paths(cat=cat))
 
-    def get_data_files(self, cat: str | None = None) -> typing.List[DataFile]:
-        """Returns the list of matched files as DataFile objects.
-        """
-
+    def get_data_files(self, cat: str | None = None) -> list[DataFile]:
+        """Returns the list of matched files as DataFile objects."""
         files = self.get_file_paths(cat=cat)
         filename_pattern = FilenamePattern(self.filename_regex)
 
-        data_files: typing.List[DataFile] = []
+        data_files: list[DataFile] = []
 
         for f in files:
             try:
-                data_files.append(DataFile.make_from_bundle(
-                    path=f, bundle_path=self._bundle_file,
-                    pattern=filename_pattern))
-            except (IOError, ValueError) as e:
+                data_files.append(
+                    DataFile.make_from_bundle(
+                        path=f,
+                        bundle_path=self._bundle_file,
+                        pattern=filename_pattern,
+                    )
+                )
+            except (OSError, ValueError) as e:
                 logger.error("Error processing file %s: %s", f, str(e))
 
         return data_files
 
-    def get_file_paths(self, cat: str | None = None) -> typing.Sequence[str]:
-        """Returns the list of matched files.
-        """
+    def get_file_paths(self, cat: str | None = None) -> typing.Sequence[Path]:
+        """Returns the list of matched files."""
         files, _, _ = self._do_get_file_paths(cat=cat)
         return files
 
-    def get_unmatched_file_paths(self, cat: str | None = None
-                                 ) -> typing.Sequence[str]:
-        """Returns the list of unmatched files.
-        """
+    def get_unmatched_file_paths(
+        self, cat: str | None = None
+    ) -> typing.Sequence[Path]:
+        """Returns the list of unmatched files."""
         _, files, _ = self._do_get_file_paths(cat=cat)
         return files
 
-    def get_all_categories(self) -> typing.Set[str]:
+    def get_all_categories(self) -> set[str]:
         """Returns all categories of the analysis."""
         _, _, categories = self._do_get_file_paths()
         return categories
 
     @staticmethod
-    def _resolve_data_path(root_dir: str, data_path: str | None) -> str:
-        resolved_data_path = '' if data_path is None else data_path
+    def _resolve_data_path(root_dir: Path, data_path: Path | None) -> Path:
+        resolved_data_path = Path() if data_path is None else data_path
 
-        if not os.path.isabs(resolved_data_path):
-            resolved_data_path = os.path.abspath(os.path.join(root_dir, resolved_data_path))
+        if not resolved_data_path.is_absolute():
+            resolved_data_path = (root_dir / resolved_data_path).absolute()
 
         return resolved_data_path
 
-    def _get_files_with_allowed_categories(self) -> typing.Dict[str, str]:
+    def _get_files_with_allowed_categories(self) -> dict[Path, str]:
         # Create a dict to store allowed files. Keys are the filepaths,
         # and values are their corresponding category.
-        allowed_files: typing.Dict[str, str] = {}
+        allowed_files: dict[Path, str] = {}
         # If files are listed explicitly in the YAML in the 'files' attribute, process them.
         if self._files is not None:
             abs_filepaths = [Path(self._data_path) / f for f in self._files]
@@ -151,14 +152,14 @@ class Analysis:
                 cat = file.suffixes[0][1:]
                 # Add each matching file and its category to the dict.
                 if cat in ALLOWED_CATEGORIES:
-                    allowed_files[str(file)] = cat
+                    allowed_files[file] = cat
         # Else, look for files on disk using the YAML 'data_path' attribute.
         else:
             # Try to retrieve files matching allowed categories using glob.
             for cat in ALLOWED_CATEGORIES:
-                glob_res = []
-                glob_res.extend(glob.glob(os.path.join(self._data_path, f"*.{cat}")))
-                glob_res.extend(glob.glob(os.path.join(self._data_path, f"*.{cat}.gz")))
+                glob_res: list[Path] = []
+                glob_res.extend(self._data_path.glob(f"*.{cat}"))
+                glob_res.extend(self._data_path.glob(f"*.{cat}.gz"))
 
                 # Add each globed file and its category to the dict.
                 for g_file in glob_res:
@@ -166,12 +167,13 @@ class Analysis:
 
         return allowed_files
 
-    def _do_get_file_paths(self, cat: str | None = None) \
-            -> tuple[typing.Sequence[str], typing.Sequence[str], typing.Set[str]]:
-
+    def _do_get_file_paths(
+        self, cat: str | None = None
+    ) -> tuple[typing.Sequence[Path], typing.Sequence[Path], set[str]]:
         # Raise an error if the category given as a parameter is not part of the allowed categories.
         if cat is not None and cat not in ALLOWED_CATEGORIES:
-            raise ValueError(f"Unknown category {cat}.")
+            msg = f"Unknown category {cat}."
+            raise ValueError(msg)
 
         # Obtain a dict of all files matching the allowed categories.
         allowed_files = self._get_files_with_allowed_categories()
@@ -181,16 +183,18 @@ class Analysis:
             files_to_match = allowed_files
         else:
             # A category was given as a parameter, so we match only this specific category.
-            files_to_match = dict((k, v) for (k, v) in allowed_files.items() if v == cat)
+            files_to_match = {
+                k: v for k, v in allowed_files.items() if v == cat
+            }
 
         filename_pattern = FilenamePattern(self.filename_regex)
-        matching_files: typing.List[str] = []
-        non_matching_files: typing.List[str] = []
+        matching_files: list[Path] = []
+        non_matching_files: list[Path] = []
         categories = set()
 
         # We filter files by ensuring that they match the filename pattern defined in the analysis.
         for file, category in sorted(files_to_match.items()):
-            if filename_pattern.matches_pattern(os.path.basename(file)):
+            if filename_pattern.matches_pattern(file.name):
                 matching_files.append(file)
                 logger.info("MATCHED file %s.", file)
                 # Add the file category to the categories set.