genelastic 0.6.0__tar.gz → 0.6.1__tar.gz

{genelastic-0.6.0 → genelastic-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: genelastic
-Version: 0.6.0
+Version: 0.6.1
 Summary: Generate and store genetic data into an Elasticsearch database.
 Author: CNRGH
 Author-email: Pierrick ROGER <pierrick.roger@cnrgh.fr>, Maxime BLANCHON <maxime.blanchon@cnrgh.fr>
@@ -21,7 +21,7 @@ Requires-Dist: colorlog
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Requires-Dist: mypy; extra == "tests"
-Requires-Dist: pylint<3.3,>=3.2; extra == "tests"
+Requires-Dist: pylint; extra == "tests"
 Requires-Dist: bandit; extra == "tests"
 Requires-Dist: coverage; extra == "tests"
 Requires-Dist: yamllint; extra == "tests"
@@ -30,6 +30,11 @@ Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-autoapi; extra == "docs"
 Requires-Dist: furo; extra == "docs"
+Provides-Extra: api
+Requires-Dist: flask; extra == "api"
+Requires-Dist: elasticsearch; extra == "api"
+Requires-Dist: environs; extra == "api"
+Requires-Dist: connexion[flask,swagger-ui,uvicorn]; extra == "api"
 
 # genelastic
 

{genelastic-0.6.0 → genelastic-0.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "genelastic"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
     "elasticsearch",
     "PyVCF3",
@@ -33,7 +33,7 @@ classifiers = [
 tests = [
     "pytest",
     "mypy",
-    "pylint>=3.2,<3.3",
+    "pylint",
     "bandit",
     "coverage",
     "yamllint",
@@ -45,9 +45,16 @@ docs = [
     "furo"
 ]
 
+api = [
+    "flask",
+    "elasticsearch",
+    "environs",
+    "connexion[flask,swagger-ui,uvicorn]"
+]
+
 [project.scripts]
-gen-data = "genelastic.gen_data:main"
-import = "genelastic.import_data:main"
-validate = "genelastic.validate_data:main"
-db_info = "genelastic.info:main"
-db_integrity = "genelastic.integrity:main"
+gen-data = "genelastic.import_data.gen_data:main"
+import = "genelastic.import_data.import_data:main"
+validate = "genelastic.import_data.validate_data:main"
+db_info = "genelastic.import_data.info:main"
+db_integrity = "genelastic.import_data.integrity:main"

genelastic-0.6.1/src/genelastic/api/extends/example.py

@@ -0,0 +1,7 @@
+# pylint: disable=missing-module-docstring
+from flask import jsonify, Response
+
+
+def ping_2() -> Response:
+    """Test route to verify that the server is online."""
+    return jsonify({'message': 'pong_2'})

genelastic-0.6.1/src/genelastic/api/routes.py

@@ -0,0 +1,84 @@
+# pylint: disable=missing-module-docstring
+from pathlib import Path
+from flask import jsonify, current_app, Response
+
+
+def ping() -> Response:
+    """Test route to verify that the server is online."""
+    return jsonify({'message': 'pong'})
+
+
+def list_indices() -> Response:
+    """Route to list Elasticsearch indexes."""
+    return current_app.elastic_query_conn.get_indices()  # type: ignore
+
+
+def retrieve_document(index_id: str, document_id: str) -> Response:
+    """Route to retrieve a document by its ID."""
+    document = (current_app.elastic_query_conn  # type: ignore
+                .get_document_by_id(index_id, document_id))
+    return jsonify(document)
+
+
+def list_wet_processes() -> Response:
+    """Route to list wet processes."""
+    wet_processes_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-wet_processes"
+    result = (current_app.elastic_query_conn  # type: ignore
+              .get_field_values(wet_processes_index, "proc_id"))
+    return jsonify(list(result))
+
+
+def list_bi_processes() -> Response:
+    """Route to list bi processes."""
+    bi_processes_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-bi_processes"
+    result = (current_app.elastic_query_conn  # type: ignore
+              .get_field_values(bi_processes_index, "name"))
+    return jsonify(list(result))
+
+
+def list_analyses() -> Response:
+    """Route to list analyses."""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+    result = current_app.elastic_query_conn.get_field_values(analyses_index, "path")  # type: ignore
+    filenames = [Path(path).name for path in result]
+    return jsonify(filenames)
+
+
+def list_analyses_wet_processes(proc_id: str) -> Response:
+    """Route to list analyses one of specific wet process"""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+
+    search_query = {
+        "query": {
+            "term": {
+                "metadata.wet_process.keyword": proc_id,
+            }
+        }
+    }
+    result = []
+    response = (current_app.elastic_query_conn  # type: ignore
+                .client.search(index=analyses_index, body=search_query))
+    for hit in response['hits']['hits']:
+        result.append(hit['_source']['path'])
+
+    return jsonify(result)
+
+
+def list_analyses_bi_processes(proc_id: str) -> Response:
+    """Route to list analyses one of specific bi process"""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+
+    search_query = {
+        "query": {
+            "term": {
+                "metadata.bi_process.keyword": proc_id,
+            }
+        }
+    }
+    result = []
+    response = (current_app.elastic_query_conn  # type: ignore
+                .client.search(index=analyses_index, body=search_query))
+    for hit in response['hits']['hits']:
+        result.append(hit['_source']['path'])
+
+    return jsonify(result)

genelastic-0.6.1/src/genelastic/api/server.py

@@ -0,0 +1,72 @@
+# pylint: disable=missing-module-docstring
+from typing import Any
+from pathlib import Path
+import yaml
+import connexion  # type: ignore
+from genelastic.common import ElasticQueryConn
+
+
+def load_yaml(file_path: Path) -> Any:
+    """Load a YAML file and return its content."""
+    content = None
+    with open(file_path, encoding='utf-8') as f:
+        try:
+            content = yaml.safe_load(f)
+        except yaml.YAMLError as exc:
+            raise SystemExit(exc) from exc
+    return content
+
+
+def aggregate_openapi_specs(main_spec_file: Path, additional_spec_path: Path) -> Any:
+    """Aggregate OpenAPI specifications from a main file and a directory
+    of additional specifications."""
+    main_spec = load_yaml(main_spec_file)
+    try:
+        entries = additional_spec_path.iterdir()
+    except OSError as exc:
+        raise SystemExit(exc) from exc
+
+    if not 'paths' in main_spec:
+        main_spec['paths'] = []
+
+    for entry in entries:
+        if not entry.is_file():
+            continue
+
+        if not entry.suffix in [".yml", ".yaml"]:
+            continue
+
+        content = load_yaml(entry)
+
+        if 'paths' in content:
+            main_spec['paths'].update(content['paths'])
+
+    return main_spec
+
+
+# Initialiser l'application Connexion
+connexion_app = connexion.FlaskApp(__name__)
+connexion_app.app.config.from_object('src.genelastic.api.settings.Config')
+
+# Initialiser le client Elasticsearch
+es_url = connexion_app.app.config['GENAPI_ES_URL']
+es_cert_fp = connexion_app.app.config['GENAPI_ES_CERT_FP']
+es_api_key = connexion_app.app.config['GENAPI_ES_ENCODED_API_KEY']
+
+connexion_app.app.elastic_query_conn = ElasticQueryConn(es_url, es_cert_fp, api_key=es_api_key)
+
+connexion_app.app.logger.debug("Successfully connected to Elasticsearch server: %s",
+                               connexion_app.app.elastic_query_conn.client.info())
+
+# Chemins des fichiers YAML
+main_yaml_file = Path(__file__).parents[0] / 'specification.yml'
+additional_yaml_dir = Path(__file__).parents[0] / 'extends'
+
+# Charger et combiner les fichiers YAML
+yaml_spec = aggregate_openapi_specs(main_yaml_file, additional_yaml_dir)
+
+# Ajouter la spécification vers OpenAPI
+connexion_app.add_api(yaml_spec)
+
+if __name__ == '__main__':
+    connexion_app.run(debug=True)

genelastic-0.6.1/src/genelastic/api/settings.py

@@ -0,0 +1,13 @@
+# pylint: disable=missing-module-docstring
+from environs import Env
+
+env = Env()
+env.read_env()
+
+# pylint: disable=missing-class-docstring,too-few-public-methods
+class Config:
+    # Charger toutes les variables d'environnement nécessaires
+    GENAPI_ES_URL = env.url("GENAPI_ES_URL").geturl()
+    GENAPI_ES_ENCODED_API_KEY = env.str("GENAPI_ES_ENCODED_API_KEY")
+    GENAPI_ES_INDEX_PREFIX = env.str("GENAPI_ES_INDEX_PREFIX")
+    GENAPI_ES_CERT_FP = env.str("GENAPI_ES_CERT_FP")

genelastic-0.6.1/src/genelastic/common/__init__.py

@@ -0,0 +1,12 @@
+"""Genelastic package for common code between API and import scripts."""
+from .elastic import ElasticQueryConn, ElasticImportConn
+from .types import (BundleDict, AnalysisMetaData, BioInfoProcessData, WetProcessesData,
+                    MetadataDocument, AnalysisDocument, BulkItems, ProcessDocument, Bucket)
+from .cli import add_verbose_control_args, add_es_connection_args
+from .exceptions import DBIntegrityError
+
+__all__ = ['ElasticQueryConn', 'ElasticImportConn', 'BundleDict', 'AnalysisMetaData',
+           'BioInfoProcessData', 'WetProcessesData', 'MetadataDocument', 'AnalysisDocument',
+           'BulkItems', 'ProcessDocument', 'Bucket', 'add_verbose_control_args',
+           'add_es_connection_args', 'DBIntegrityError'
+           ]

genelastic-0.6.1/src/genelastic/common/cli.py

@@ -0,0 +1,35 @@
+"""Utility functions for CLI scripts."""
+import argparse
+
+
+def add_verbose_control_args(parser: argparse.ArgumentParser) -> None:
+    """
+    Add verbose control arguments to the parser.
+    Arguments are added to the parser by using its reference.
+    """
+    parser.add_argument('-q', '--quiet', dest='verbose', action='store_const',
+                        const=0, default=1,
+                        help='Set verbosity to 0 (quiet mode).')
+    parser.add_argument('-v', '--verbose', dest='verbose', action='count',
+                        default=1,
+                        help=('Verbose level. -v for information, -vv for debug,' +
+                              ' -vvv for trace.'))
+
+
+def add_es_connection_args(parser: argparse.ArgumentParser) -> None:
+    """
+    Add arguments to the parser needed to gather ElasticSearch server connection parameters.
+    Arguments are added to the parser by using its reference.
+    """
+    parser.add_argument('--es-host', dest='es_host', default='localhost',
+                        help='Address of Elasticsearch host.')
+    parser.add_argument('--es-port', type=int, default=9200, dest='es_port',
+                        help='Elasticsearch port.')
+    parser.add_argument('--es-usr', dest='es_usr', default='elastic',
+                        help='Elasticsearch user.')
+    parser.add_argument('--es-pwd', dest='es_pwd', required=True,
+                        help='Elasticsearch password.')
+    parser.add_argument('--es-cert-fp', dest='es_cert_fp',
+                        help='Elasticsearch sha256 certificate fingerprint.')
+    parser.add_argument('--es-index-prefix', dest='es_index_prefix',
+                        help='Add the given prefix to each index created during import.')

genelastic-0.6.1/src/genelastic/common/elastic.py

@@ -0,0 +1,183 @@
+# pylint: disable=missing-module-docstring
+import datetime
+import logging
+import time
+import typing
+from abc import ABC
+from typing import Any
+
+import elastic_transport
+import elasticsearch.helpers
+from elasticsearch import Elasticsearch
+
+from .exceptions import DBIntegrityError
+from .types import Bucket, BulkItems
+
+logger = logging.getLogger('genelastic')
+
+
+class ElasticConn(ABC):  # pylint: disable=too-few-public-methods
+    """Abstract class representing a connector for an Elasticsearch server."""
+    client: Elasticsearch
+
+    def __init__(self, url: str, fingerprint: str, **kwargs: Any):
+        """Initialize an elasticsearch client instance.
+
+        :url: URL of the Elasticsearch host.
+        :fingerprint: sha256 certificate fingerprint for a secure HTTP connection.
+        :returns: The configured elasticsearch client instance.
+        :raises SystemExit: If the connection to the Elasticsearch server failed.
+        """
+        try:
+            self.client = Elasticsearch(
+                url,
+                ssl_assert_fingerprint=fingerprint,
+                # Verify cert only when the fingerprint is not None.
+                verify_certs=bool(fingerprint),
+                **kwargs
+            )
+            self.client.info()
+        except (elastic_transport.TransportError, elasticsearch.AuthenticationException) as e:
+            raise SystemExit(e) from e
+
+
+class ElasticImportConn(ElasticConn):  # pylint: disable=too-few-public-methods
+    """Connector to import data into an Elasticsearch database."""
+    def import_items(self, bulk_items: BulkItems,
+                     start_time: float,
+                     total_items: int) -> None:
+        """Import items to the Elasticsearch database."""
+        if len(bulk_items) > 0:
+            elasticsearch.helpers.bulk(self.client, bulk_items)
+        elapsed = time.perf_counter() - start_time
+        logger.info("Imported %d items in %s (%f items/s).", total_items,
+                    datetime.timedelta(seconds=elapsed), total_items / elapsed)
+
+
+class ElasticQueryConn(ElasticConn):
+    """Connector to query data from an Elasticsearch database."""
+
+    def get_indices(self) -> Any | str:
+        """Return all indices."""
+        return self.client.cat.indices(format="json").body
+
+    def get_document_by_id(self, index: str, document_id: str) -> Any | str:
+        """Return a document by its ID."""
+        return self.client.get(index=index, id=document_id).body
+
+    def run_composite_aggregation(self, index: str, query: dict[str, typing.Any]) \
+            -> list[Bucket]:
+        """
+        Executes a composite aggregation on an Elasticsearch index and
+        returns all paginated results.
+
+        :param index: Name of the index to query.
+        :param query: Aggregation query to run.
+        :return: List of aggregation results.
+        """
+        # Extract the aggregation name from the query dict.
+        agg_name = next(iter(query["aggs"]))
+        all_buckets: typing.List[Bucket] = []
+
+        try:
+            logger.debug("Running composite aggregation query %s on index '%s'.", query, index)
+            response = self.client.search(index=index, body=query)
+        except elasticsearch.NotFoundError as e:
+            raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
+
+        while True:
+            # Extract buckets from the response.
+            buckets: typing.List[Bucket] = response['aggregations'][agg_name]['buckets']
+            all_buckets.extend(buckets)
+
+            # Check if there are more results to fetch.
+            if 'after_key' in response['aggregations'][agg_name]:
+                after_key = response['aggregations'][agg_name]['after_key']
+                query['aggs'][agg_name]['composite']['after'] = after_key
+                try:
+                    logger.debug("Running query %s on index '%s'.", query, index)
+                    # Fetch the next page of results.
+                    response = self.client.search(index=index, body=query)
+                except elasticsearch.NotFoundError as e:
+                    raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
+            else:
+                break
+
+        return all_buckets
+
+    def get_field_values(self, index: str, field_name: str) -> set[str]:
+        """Return a set of values for a given field."""
+        values = set()
+
+        query = {
+            "size": 0,
+            "aggs": {
+                "get_field_values": {
+                    "composite": {
+                        "sources": {"values": {"terms": {"field": f"{field_name}.keyword"}}},
+                        "size": 1000,
+                    }
+                }
+            }
+        }
+
+        buckets: typing.List[Bucket] = self.run_composite_aggregation(index, query)
+
+        for bucket in buckets:
+            values.add(bucket['key']['values'])
+
+        return values
+
+    def search_by_field_value(self, index: str, field: str, value: str) -> (
+            typing.Dict[str, typing.Any] | None):
+        """Search a document by a value for a certain field."""
+        logger.info("Searching for field '%s' with value '%s' inside index '%s'.",
+                    field, value, index)
+        search_query = {
+            "query": {
+                "term": {
+                    f"{field}.keyword": value,
+                }
+            }
+        }
+
+        response = self.client.search(index=index, body=search_query)
+
+        try:
+            return response['hits']['hits'][0]['_source']  # type: ignore
+        except KeyError:
+            return None
+
+    def ensure_unique(self, index: str, field: str) -> None:
+        """
+        Ensure that all values of a field in an index are all unique.
+
+        :param index: Name of the index.
+        :param field: Field name to check for value uniqueness.
+        :raises genelastic.common.DBIntegrityError:
+            Some values of the given field are duplicated in the index.
+        """
+
+        logger.info("Ensuring that the field '%s' in the index '%s' only contains unique values...",
+                    field, index)
+        query = {
+            "size": 0,
+            "aggs": {
+                "duplicate_proc_ids": {
+                    "terms": {
+                        "field": f"{field}.keyword",
+                        "size": 10000,
+                        "min_doc_count": 2
+                    }
+                }
+            }
+        }
+        buckets: typing.List[Bucket] = self.run_composite_aggregation(index, query)
+        duplicated_processes: typing.Set[str] = set(map(lambda bucket: str(bucket["key"]), buckets))
+
+        if len(duplicated_processes) > 0:
+            raise DBIntegrityError(f"Found non-unique value for field {field} in index '{index}': "
+                                   f"{', '.join(duplicated_processes)}.")
+
+        logger.info("All values of field '%s' in index '%s' are unique.",
+                    field, index)

genelastic-0.6.1/src/genelastic/common/exceptions.py

@@ -0,0 +1,6 @@
+# pylint: disable=missing-module-docstring
+
+class DBIntegrityError(Exception):
+    """Represents an integrity error,
+    raised when the database content does not match the expected data schema.
+    """

genelastic-0.6.1/src/genelastic/common/types.py

@@ -0,0 +1,20 @@
+# pylint: disable=missing-module-docstring
+
+import typing
+
+Bucket: typing.TypeAlias = dict[str, dict[typing.Any, typing.Any]]
+
+AnalysisMetaData: typing.TypeAlias = typing.Dict[str, str | int]
+WetProcessesData: typing.TypeAlias = typing.Dict[str, str | int | float]
+BioInfoProcessData: typing.TypeAlias = typing.Dict[str, str | typing.List[str]]
+BundleDict: typing.TypeAlias = typing.Dict[str, typing.Any]
+
+AnalysisDocument: typing.TypeAlias = typing.Dict[str, str | None | AnalysisMetaData]
+MetadataDocument: typing.TypeAlias = typing.Dict[str, int | str | typing.List[typing.Any | None]]
+ProcessDocument: typing.TypeAlias = (typing.Dict[str, str] |
+                                     WetProcessesData |
+                                     BioInfoProcessData)
+BulkItems: typing.TypeAlias = typing.List[typing.Dict[str, str |
+                                                           MetadataDocument |
+                                                           AnalysisDocument |
+                                                           ProcessDocument]]

genelastic-0.6.1/src/genelastic/import_data/__init__.py

@@ -0,0 +1,9 @@
+"""Genelastic package for importing Genomic data into Elasticsearch."""
+from .analysis import Analysis
+from .import_bundle_factory import (make_import_bundle_from_files,
+                                    load_import_bundle_file)
+from .tags import Tags
+from .import_bundle import ImportBundle
+
+__all__ = ['Analysis', 'Tags', 'ImportBundle', 'make_import_bundle_from_files',
+           'load_import_bundle_file']

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/analyses.py

@@ -1,7 +1,9 @@
 # pylint: disable=missing-module-docstring
 import typing
+
+from  genelastic.common import BundleDict
+
 from .analysis import Analysis
-from .common import BundleDict
 from .data_file import DataFile
 
 class Analyses:

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/analysis.py

@@ -7,7 +7,8 @@ import re
 import typing
 from pathlib import Path
 
-from .common import AnalysisMetaData
+from genelastic.common import AnalysisMetaData
+
 from .constants import ALLOWED_CATEGORIES
 from .data_file import DataFile
 from .filename_pattern import FilenamePattern
@@ -19,7 +20,7 @@ logger = logging.getLogger('genelastic')
 class Analysis:
     """Class Analysis that represents an analysis."""
 
-    # pylint: disable-next=too-many-arguments
+    # pylint: disable-next=too-many-arguments, too-many-positional-arguments
     def __init__(self,
                  tags: Tags,
                  root_dir: str = '.',

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/bi_process.py

@@ -2,7 +2,7 @@
 import copy
 import typing
 
-from .common import BioInfoProcessData
+from genelastic.common import BioInfoProcessData
 
 
 class BioInfoProcess:

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/bi_processes.py

@@ -2,8 +2,9 @@
 import logging
 import typing
 
+from genelastic.common import BundleDict
+
 from .bi_process import BioInfoProcess
-from .common import BundleDict
 
 logger = logging.getLogger('genelastic')
 

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/data_file.py

@@ -11,8 +11,10 @@ import logging
 import os
 import pathlib
 import typing
+
+from genelastic.common import AnalysisMetaData
+
 from .filename_pattern import FilenamePattern
-from .common import AnalysisMetaData
 
 logger = logging.getLogger('genelastic')
 

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/filename_pattern.py

@@ -5,7 +5,8 @@ and extract metadata from file names using this pattern.
 
 import logging
 import re
-from .common import AnalysisMetaData
+
+from genelastic.common import AnalysisMetaData
 
 logger = logging.getLogger('genelastic')
 

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/gen_data.py

@@ -8,8 +8,8 @@ import sys
 from typing import Dict, List, Sequence, Collection
 
 import yaml
-
 from genelastic.common import add_verbose_control_args
+
 from .logger import configure_logging
 
 logger = logging.getLogger('genelastic')
@@ -19,7 +19,8 @@ def read_args() -> argparse.Namespace:
     # pylint: disable=R0801
     """Read arguments from command line."""
     parser = argparse.ArgumentParser(description='Genetics data random generator.',
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+                                     allow_abbrev=False)
     add_verbose_control_args(parser)
     parser.add_argument('-d', '--data-folder', dest='data_folder', required=True,
                         help='Data destination folder.')

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/import_bundle.py

@@ -8,9 +8,10 @@ import logging
 import sys
 import typing
 
+from genelastic.common import BundleDict
+
 from .bi_processes import BioInfoProcesses
 from .data_file import DataFile
-from .common import BundleDict
 from .constants import BUNDLE_CURRENT_VERSION
 from .analyses import Analyses
 from .tags import Tags

{genelastic-0.6.0/src/genelastic → genelastic-0.6.1/src/genelastic/import_data}/import_bundle_factory.py

@@ -11,8 +11,10 @@ from yaml.scanner import ScannerError
 
 import schema  # type: ignore[import-untyped]
 import yaml
+
+from genelastic.common import BundleDict
+
 from .import_bundle import ImportBundle
-from .common import BundleDict
 from .constants import BUNDLE_CURRENT_VERSION
 
 logger = logging.getLogger('genelastic')