genelastic 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

genelastic/__init__.py

@@ -1,13 +0,0 @@
-"""Genelastic package for importing Genomic data into Elasticsearch.
-"""
-
-from .import_bundle import ImportBundle
-from .common import BundleDict
-from .constants import BUNDLE_CURRENT_VERSION
-from .import_bundle_factory import make_import_bundle_from_files, \
-    load_import_bundle_file
-from .analysis import Analysis
-from .analyses import Analyses
-
-__all__ = ['make_import_bundle_from_files', 'BUNDLE_CURRENT_VERSION',
-           'load_import_bundle_file', 'Analysis', 'ImportBundle']

genelastic/api/extends/example.py

@@ -0,0 +1,6 @@
+from flask import Response, jsonify
+
+
+def ping_2() -> Response:
+    """Test route to verify that the server is online."""
+    return jsonify({"message": "pong_2"})

genelastic/api/routes.py

@@ -0,0 +1,221 @@
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any
+
+from flask import Response, current_app, jsonify
+
+
+def ping() -> Response:
+    """Test route to verify that the server is online."""
+    return jsonify({"message": "pong"})
+
+
+def list_indices() -> Response:
+    """Route to list Elasticsearch indexes."""
+    return current_app.elastic_query_conn.get_indices()  # type: ignore[attr-defined, no-any-return]
+
+
+def retrieve_document(index_id: str, document_id: str) -> Response:
+    """Route to retrieve a document by its ID."""
+    document = current_app.elastic_query_conn.get_document_by_id(  # type: ignore[attr-defined]
+        index_id, document_id
+    )
+    return jsonify(document)
+
+
+def list_wet_processes() -> Response:
+    """Route to list wet processes."""
+    wet_processes_index = (
+        f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-wet_processes"
+    )
+    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
+        wet_processes_index, "proc_id"
+    )
+    return jsonify(list(result))
+
+
+def list_bi_processes() -> Response:
+    """Route to list bi processes."""
+    bi_processes_index = (
+        f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-bi_processes"
+    )
+    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
+        bi_processes_index, "proc_id"
+    )
+    return jsonify(list(result))
+
+
+def list_analyses() -> Response:
+    """Route to list analyses."""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
+        analyses_index, "path"
+    )
+    filenames = [Path(path).name for path in result]
+    return jsonify(filenames)
+
+
+def list_analyses_wet_processes(proc_id: str) -> Response:
+    """Route to list analyses one of specific wet process"""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+
+    search_query = {
+        "query": {
+            "term": {
+                "metadata.wet_process.keyword": proc_id,
+            }
+        }
+    }
+    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+        index=analyses_index, body=search_query
+    )
+    result = [hit["_source"]["path"] for hit in response["hits"]["hits"]]
+
+    return jsonify(result)
+
+
+def list_analyses_bi_processes(proc_id: str) -> Response:
+    """Route to list analyses one of specific bi process"""
+    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
+
+    search_query = {
+        "query": {
+            "term": {
+                "metadata.bi_process.keyword": proc_id,
+            }
+        }
+    }
+    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+        index=analyses_index, body=search_query
+    )
+    result = [hit["_source"]["path"] for hit in response["hits"]["hits"]]
+
+    return jsonify(result)
+
+
+def list_snv_documents() -> Response:
+    """Route to list all documents containing a mutation at a single position (SNV)."""
+    index_pattern = "genelastic-file-*"
+    target_value = "SNV"
+
+    search_query = {
+        "aggs": {
+            "snv_docs": {
+                "composite": {
+                    "sources": [
+                        {"alt_value": {"terms": {"field": "alt.keyword"}}}
+                    ],
+                    "size": 1000,
+                }
+            }
+        },
+        "query": {"term": {"alt.keyword": target_value}},
+        "size": 0,
+    }
+
+    all_documents = []
+    buckets = current_app.elastic_query_conn.run_composite_aggregation(  # type: ignore[attr-defined]
+        index_pattern, search_query
+    )
+
+    for bucket in buckets:
+        alt_value = bucket["key"]["alt_value"]
+
+        search_query_docs = {
+            "query": {"term": {"alt.keyword": alt_value}},
+            "size": 1000,
+        }
+
+        response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+            index=index_pattern, body=search_query_docs
+        )
+
+        all_documents.extend(response["hits"]["hits"])
+
+    return jsonify(all_documents)
+
+
+def build_snv_search_query(
+    target_alt: str, target_svtype: str
+) -> dict[str, Any]:
+    """Helper function to build the search query for SNV documents with specified alt and SVTYPE."""
+    return {
+        "query": {
+            "bool": {
+                "must": [
+                    {"term": {"alt.keyword": target_alt}},
+                    {"term": {"info.SVTYPE.keyword": target_svtype}},
+                ]
+            }
+        },
+        "size": 1000,
+    }
+
+
+def build_snv_mutation_search_query(
+    target_svtypes: list[str],
+) -> dict[str, Any]:
+    """Helper function to build the search query for SNV mutations with specified SVTYPE values."""
+    return {
+        "query": {
+            "bool": {
+                "must": [
+                    {"term": {"alt.keyword": "SNV"}},
+                    {"terms": {"info.SVTYPE.keyword": target_svtypes}},
+                ]
+            }
+        },
+        "size": 1000,
+    }
+
+
+def list_snv_insertion_documents() -> Response:
+    """Route to list all documents containing an insertion (INS) at a single position (SNV)."""
+    index_pattern = "genelastic-file-*"
+    search_query = build_snv_search_query(target_alt="SNV", target_svtype="INS")
+
+    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+        index=index_pattern, body=search_query
+    )
+
+    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+
+    return jsonify(all_documents)
+
+
+def list_snv_deletion_documents() -> Response:
+    """Route to list all documents containing a deletion (DEL) at a single position (SNV)."""
+    index_pattern = "genelastic-file-*"
+    search_query = build_snv_search_query(target_alt="SNV", target_svtype="DEL")
+
+    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+        index=index_pattern, body=search_query
+    )
+
+    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+
+    return jsonify(all_documents)
+
+
+def list_snv_mutation_documents() -> Response:
+    """Route to list all documents containing a mutation at a single position (SNV)."""
+    index_pattern = "genelastic-file-*"
+    target_svtypes = ["INS", "DEL"]
+
+    search_query = build_snv_mutation_search_query(
+        target_svtypes=target_svtypes
+    )
+
+    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
+        index=index_pattern, body=search_query
+    )
+
+    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+
+    return jsonify(all_documents)
+
+
+def get_genelastic_version() -> Response:
+    """Retourne la version du package genelastic."""
+    top_level_package = __package__.split(".")[0]
+    return jsonify({"version": version(top_level_package)})

genelastic/api/server.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+from typing import Any
+
+import connexion
+import yaml
+
+from genelastic.common import ElasticQueryConn
+
+
+def load_yaml(file_path: Path) -> Any:  # noqa: ANN401
+    """Load a YAML file and return its content."""
+    content = None
+    with Path.open(file_path, encoding="utf-8") as f:
+        try:
+            content = yaml.safe_load(f)
+        except yaml.YAMLError as exc:
+            raise SystemExit(exc) from exc
+    return content
+
+
+def aggregate_openapi_specs(
+    main_spec_file: Path, additional_spec_path: Path
+) -> Any:  # noqa: ANN401
+    """Aggregate OpenAPI specifications from a main file and a directory
+    of additional specifications.
+    """
+    main_spec = load_yaml(main_spec_file)
+    try:
+        entries = additional_spec_path.iterdir()
+    except OSError as exc:
+        raise SystemExit(exc) from exc
+
+    if "paths" not in main_spec:
+        main_spec["paths"] = []
+
+    for entry in entries:
+        if not entry.is_file():
+            continue
+
+        if entry.suffix not in [".yml", ".yaml"]:
+            continue
+
+        content = load_yaml(entry)
+
+        if "paths" in content:
+            main_spec["paths"].update(content["paths"])
+
+    return main_spec
+
+
+# Initialiser l'application Connexion
+connexion_app = connexion.FlaskApp(__name__)
+connexion_app.app.config.from_object("src.genelastic.api.settings.Config")
+
+# Initialiser le client Elasticsearch
+es_url = connexion_app.app.config["GENAPI_ES_URL"]
+es_cert_fp = connexion_app.app.config["GENAPI_ES_CERT_FP"]
+es_api_key = connexion_app.app.config["GENAPI_ES_ENCODED_API_KEY"]
+
+connexion_app.app.elastic_query_conn = ElasticQueryConn(
+    es_url, es_cert_fp, api_key=es_api_key
+)
+
+connexion_app.app.logger.debug(
+    "Successfully connected to Elasticsearch server: %s",
+    connexion_app.app.elastic_query_conn.client.info(),
+)
+
+# Chemins des fichiers YAML
+main_yaml_file = Path(__file__).parents[0] / "specification.yml"
+additional_yaml_dir = Path(__file__).parents[0] / "extends"
+
+# Charger et combiner les fichiers YAML
+yaml_spec = aggregate_openapi_specs(main_yaml_file, additional_yaml_dir)
+
+# Ajouter la spécification vers OpenAPI
+connexion_app.add_api(yaml_spec)
+
+if __name__ == "__main__":
+    connexion_app.run(debug=True)

genelastic/api/settings.py

@@ -0,0 +1,14 @@
+from environs import Env
+
+env = Env()
+env.read_env()
+
+
+class Config:
+    """Flask config class."""
+
+    # Charger toutes les variables d'environnement nécessaires
+    GENAPI_ES_URL = env.url("GENAPI_ES_URL").geturl()
+    GENAPI_ES_ENCODED_API_KEY = env.str("GENAPI_ES_ENCODED_API_KEY")
+    GENAPI_ES_INDEX_PREFIX = env.str("GENAPI_ES_INDEX_PREFIX")
+    GENAPI_ES_CERT_FP = env.str("GENAPI_ES_CERT_FP")

genelastic/common/__init__.py

@@ -0,0 +1,39 @@
+"""Genelastic package for common code between API and import scripts."""
+
+from .cli import add_es_connection_args, add_verbose_control_args
+from .elastic import ElasticImportConn, ElasticQueryConn
+from .exceptions import DBIntegrityError
+from .types import (
+    AnalysisDocument,
+    AnalysisMetaData,
+    BioInfoProcessData,
+    Bucket,
+    BulkItems,
+    BundleDict,
+    MetadataDocument,
+    ProcessDocument,
+    RandomAnalysisData,
+    RandomBiProcessData,
+    RandomWetProcessData,
+    WetProcessesData,
+)
+
+__all__ = [
+    "AnalysisDocument",
+    "AnalysisMetaData",
+    "BioInfoProcessData",
+    "Bucket",
+    "BulkItems",
+    "BundleDict",
+    "DBIntegrityError",
+    "ElasticImportConn",
+    "ElasticQueryConn",
+    "MetadataDocument",
+    "ProcessDocument",
+    "RandomAnalysisData",
+    "RandomBiProcessData",
+    "RandomWetProcessData",
+    "WetProcessesData",
+    "add_es_connection_args",
+    "add_verbose_control_args",
+]

genelastic/common/cli.py

@@ -0,0 +1,63 @@
+"""Utility functions for CLI scripts."""
+
+import argparse
+
+
+def add_verbose_control_args(parser: argparse.ArgumentParser) -> None:
+    """Add verbose control arguments to the parser.
+    Arguments are added to the parser by using its reference.
+    """
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="verbose",
+        action="store_const",
+        const=0,
+        default=1,
+        help="Set verbosity to 0 (quiet mode).",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbose",
+        action="count",
+        default=1,
+        help=(
+            "Verbose level. -v for information, -vv for debug, -vvv for trace."
+        ),
+    )
+
+
+def add_es_connection_args(parser: argparse.ArgumentParser) -> None:
+    """Add arguments to the parser needed to gather ElasticSearch server connection parameters.
+    Arguments are added to the parser by using its reference.
+    """
+    parser.add_argument(
+        "--es-host",
+        dest="es_host",
+        default="localhost",
+        help="Address of Elasticsearch host.",
+    )
+    parser.add_argument(
+        "--es-port",
+        type=int,
+        default=9200,
+        dest="es_port",
+        help="Elasticsearch port.",
+    )
+    parser.add_argument(
+        "--es-usr", dest="es_usr", default="elastic", help="Elasticsearch user."
+    )
+    parser.add_argument(
+        "--es-pwd", dest="es_pwd", required=True, help="Elasticsearch password."
+    )
+    parser.add_argument(
+        "--es-cert-fp",
+        dest="es_cert_fp",
+        help="Elasticsearch sha256 certificate fingerprint.",
+    )
+    parser.add_argument(
+        "--es-index-prefix",
+        dest="es_index_prefix",
+        help="Add the given prefix to each index created during import.",
+    )

genelastic/common/elastic.py

@@ -0,0 +1,214 @@
+import datetime
+import logging
+import time
+import typing
+from abc import ABC
+from typing import Any
+
+import elastic_transport
+import elasticsearch.helpers
+from elasticsearch import Elasticsearch
+
+from .exceptions import DBIntegrityError
+from .types import Bucket, BulkItems
+
+logger = logging.getLogger("genelastic")
+
+
+class ElasticConn(ABC):
+    """Abstract class representing a connector for an Elasticsearch server."""
+
+    client: Elasticsearch
+
+    def __init__(self, url: str, fingerprint: str, **kwargs: Any) -> None:  # noqa: ANN401
+        """Initialize an elasticsearch client instance.
+
+        :url: URL of the Elasticsearch host.
+        :fingerprint: sha256 certificate fingerprint for a secure HTTP connection.
+        :returns: The configured elasticsearch client instance.
+        :raises SystemExit: If the connection to the Elasticsearch server failed.
+        """
+        try:
+            self.client = Elasticsearch(
+                url,
+                ssl_assert_fingerprint=fingerprint,
+                # Verify cert only when the fingerprint is not None.
+                verify_certs=bool(fingerprint),
+                **kwargs,
+            )
+            self.client.info()
+        except (
+            elastic_transport.TransportError,
+            elasticsearch.AuthenticationException,
+        ) as e:
+            raise SystemExit(e) from e
+
+
+class ElasticImportConn(ElasticConn):
+    """Connector to import data into an Elasticsearch database."""
+
+    def import_items(
+        self, bulk_items: BulkItems, start_time: float, total_items: int
+    ) -> None:
+        """Import items to the Elasticsearch database."""
+        if len(bulk_items) > 0:
+            elasticsearch.helpers.bulk(self.client, bulk_items)
+        elapsed = time.perf_counter() - start_time
+        logger.info(
+            "Imported %d items in %s (%f items/s).",
+            total_items,
+            datetime.timedelta(seconds=elapsed),
+            total_items / elapsed,
+        )
+
+
+class ElasticQueryConn(ElasticConn):
+    """Connector to query data from an Elasticsearch database."""
+
+    def get_indices(self) -> Any | str:  # noqa: ANN401
+        """Return all indices."""
+        return self.client.cat.indices(format="json").body
+
+    def get_document_by_id(self, index: str, document_id: str) -> Any | str:  # noqa: ANN401
+        """Return a document by its ID."""
+        return self.client.get(index=index, id=document_id).body
+
+    def run_composite_aggregation(
+        self, index: str, query: dict[str, typing.Any]
+    ) -> list[Bucket]:
+        """Executes a composite aggregation on an Elasticsearch index and
+        returns all paginated results.
+
+        :param index: Name of the index to query.
+        :param query: Aggregation query to run.
+        :return: List of aggregation results.
+        """
+        # Extract the aggregation name from the query dict.
+        agg_name = next(iter(query["aggs"]))
+        all_buckets: list[Bucket] = []
+
+        try:
+            logger.debug(
+                "Running composite aggregation query %s on index '%s'.",
+                query,
+                index,
+            )
+            response = self.client.search(index=index, body=query)
+        except elasticsearch.NotFoundError as e:
+            msg = f"Error: {e.message} for index '{index}'."
+            raise SystemExit(msg) from e
+
+        while True:
+            # Extract buckets from the response.
+            buckets: list[Bucket] = response["aggregations"][agg_name][
+                "buckets"
+            ]
+            all_buckets.extend(buckets)
+
+            # Check if there are more results to fetch.
+            if "after_key" in response["aggregations"][agg_name]:
+                after_key = response["aggregations"][agg_name]["after_key"]
+                query["aggs"][agg_name]["composite"]["after"] = after_key
+                try:
+                    logger.debug(
+                        "Running query %s on index '%s'.", query, index
+                    )
+                    # Fetch the next page of results.
+                    response = self.client.search(index=index, body=query)
+                except elasticsearch.NotFoundError as e:
+                    msg = f"Error: {e.message} for index '{index}'."
+                    raise SystemExit(msg) from e
+            else:
+                break
+
+        return all_buckets
+
+    def get_field_values(self, index: str, field_name: str) -> set[str]:
+        """Return a set of values for a given field."""
+        values = set()
+
+        query = {
+            "size": 0,
+            "aggs": {
+                "get_field_values": {
+                    "composite": {
+                        "sources": {
+                            "values": {
+                                "terms": {"field": f"{field_name}.keyword"}
+                            }
+                        },
+                        "size": 1000,
+                    }
+                }
+            },
+        }
+
+        buckets: list[Bucket] = self.run_composite_aggregation(index, query)
+
+        for bucket in buckets:
+            values.add(bucket["key"]["values"])
+
+        return values
+
+    def search_by_field_value(
+        self, index: str, field: str, value: str
+    ) -> dict[str, typing.Any] | None:
+        """Search a document by a value for a certain field."""
+        logger.info(
+            "Searching for field '%s' with value '%s' inside index '%s'.",
+            field,
+            value,
+            index,
+        )
+        search_query = {
+            "query": {
+                "term": {
+                    f"{field}.keyword": value,
+                }
+            }
+        }
+
+        response = self.client.search(index=index, body=search_query)
+
+        try:
+            return response["hits"]["hits"][0]["_source"]  # type: ignore[no-any-return]
+        except KeyError:
+            return None
+
+    def ensure_unique(self, index: str, field: str) -> None:
+        """Ensure that all values of a field in an index are all unique.
+
+        :param index: Name of the index.
+        :param field: Field name to check for value uniqueness.
+        :raises genelastic.common.DBIntegrityError:
+            Some values of the given field are duplicated in the index.
+        """
+        logger.info(
+            "Ensuring that the field '%s' in the index '%s' only contains unique values...",
+            field,
+            index,
+        )
+        query = {
+            "size": 0,
+            "aggs": {
+                "duplicate_proc_ids": {
+                    "terms": {
+                        "field": f"{field}.keyword",
+                        "size": 10000,
+                        "min_doc_count": 2,
+                    }
+                }
+            },
+        }
+        buckets: list[Bucket] = self.run_composite_aggregation(index, query)
+        duplicated_processes: set[str] = {
+            str(bucket["key"]) for bucket in buckets
+        }
+
+        if len(duplicated_processes) > 0:
+            msg = f"Found non-unique value for field {field} in index '{index}': {', '.join(duplicated_processes)}."
+            raise DBIntegrityError(msg)
+
+        logger.info(
+            "All values of field '%s' in index '%s' are unique.", field, index
+        )

genelastic/common/exceptions.py

@@ -0,0 +1,4 @@
+class DBIntegrityError(Exception):
+    """Represents an integrity error,
+    raised when the database content does not match the expected data schema.
+    """

genelastic/common/types.py

@@ -0,0 +1,25 @@
+import typing
+
+# Types related to Elasticsearch data import.
+Bucket: typing.TypeAlias = dict[str, dict[typing.Any, typing.Any]]
+BundleDict: typing.TypeAlias = dict[str, typing.Any]
+
+AnalysisMetaData: typing.TypeAlias = dict[str, str | int]
+WetProcessesData: typing.TypeAlias = dict[str, str | int | float]
+BioInfoProcessData: typing.TypeAlias = dict[str, str | list[str]]
+
+AnalysisDocument: typing.TypeAlias = dict[str, str | None | AnalysisMetaData]
+MetadataDocument: typing.TypeAlias = dict[
+    str, int | str | list[typing.Any | None]
+]
+ProcessDocument: typing.TypeAlias = (
+    dict[str, str] | WetProcessesData | BioInfoProcessData
+)
+BulkItems: typing.TypeAlias = list[
+    dict[str, str | MetadataDocument | AnalysisDocument | ProcessDocument]
+]
+
+# Types related to random bundle generation.
+RandomBiProcessData: typing.TypeAlias = dict[str, str | list[dict[str, str]]]
+RandomWetProcessData: typing.TypeAlias = dict[str, str | float]
+RandomAnalysisData: typing.TypeAlias = dict[str, str | list[int | str]]