genelastic 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/api/.env

@@ -0,0 +1,4 @@
+GENAPI_ES_URL="https://192.168.73.11:9200"
+GENAPI_ES_CERT_FP="CB:8F:94:61:DB:57:65:38:9E:A2:1E:C4:75:AA:3E:9B:03:5C:2A:04:74:06:96:3E:B5:CE:81:7C:64:3C:D4:C5"
+GENAPI_ES_ENCODED_API_KEY="YlNSWjY1Z0JoN1BKQ1BHTW95OGI6VExoTUxPc1VaRVFJazRnMWdwdGtNdw=="
+GENAPI_ES_INDEX_PREFIX="genelastic"

genelastic/api/cli_start_api.py

@@ -0,0 +1,18 @@
+from genelastic.common.cli import parse_server_launch_args
+from genelastic.common.server import start_dev_server, start_prod_server
+
+
+def main() -> None:
+    app_module = "genelastic.api.server:app"
+    args = parse_server_launch_args("Start API server.", 8000)
+    if args.env == "dev":
+        start_dev_server(app_module, args, reload_includes=["*.yml"])
+    elif args.env == "prod":
+        start_prod_server(app_module, args)
+    else:
+        msg = f"Environment '{args.env}' is not implemented."
+        raise NotImplementedError(msg)
+
+
+if __name__ == "__main__":
+    main()

genelastic/api/errors.py

@@ -0,0 +1,52 @@
+import elasticsearch
+from flask import Flask, Response, jsonify
+
+
+def register_error_handlers(app: Flask) -> None:
+    """Register handlers for elasticsearch ApiError and TransportError
+    exceptions.
+    """
+
+    @app.errorhandler(elasticsearch.ApiError)
+    def handle_api_error(e: elasticsearch.ApiError) -> tuple[Response, int]:
+        # See https://elasticsearch-py.readthedocs.io/en/latest/exceptions.html#api-errors
+        app.logger.error("Elasticsearch ApiError: %s", e, exc_info=True)
+
+        return (
+            jsonify(
+                {
+                    "error": {
+                        "message": str(e),
+                        "type": e.__class__.__name__,
+                    }
+                }
+            ),
+            e.status_code,
+        )
+
+    @app.errorhandler(elasticsearch.TransportError)
+    def handle_transport_error(
+        e: elasticsearch.TransportError,
+    ) -> tuple[Response, int]:
+        # See https://elasticsearch-py.readthedocs.io/en/latest/exceptions.html#transport-and-connection-errors
+        app.logger.error("Elasticsearch TransportError: %s", e, exc_info=True)
+
+        status_code = 502
+        if (
+            e.errors
+            and hasattr(e.errors[0], "status")
+            and e.errors[0].status is not None
+        ):
+            status_code = e.errors[0].status
+
+        return (
+            jsonify(
+                {
+                    "error": {
+                        "message": str(e),
+                        "type": e.__class__.__name__,
+                    }
+                }
+            ),
+            status_code,
+        )

genelastic/api/extends/example.py

@@ -1,6 +0,0 @@
-from flask import Response, jsonify
-
-
-def ping_2() -> Response:
-    """Test route to verify that the server is online."""
-    return jsonify({"message": "pong_2"})

genelastic/api/routes.py

@@ -1,221 +1,353 @@
-from importlib.metadata import version
-from pathlib import Path
-from typing import Any
+import typing
+from importlib.metadata import version as get_version
 
 from flask import Response, current_app, jsonify
 
+if typing.TYPE_CHECKING:
+    from genelastic.common.elastic import ElasticQueryConn
 
-def ping() -> Response:
-    """Test route to verify that the server is online."""
-    return jsonify({"message": "pong"})
 
+def indexes() -> tuple[Response, int]:
+    """List all Elasticsearch indexes."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    return jsonify({"result": es_query_conn.get_indices()}), 200
 
-def list_indices() -> Response:
-    """Route to list Elasticsearch indexes."""
-    return current_app.elastic_query_conn.get_indices()  # type: ignore[attr-defined, no-any-return]
 
+def retrieve_document_by_index_and_id(
+    index_id: str, document_id: str
+) -> tuple[Response, int]:
+    """Retrieve a document by its index name and ID."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.get_document_by_id(index_id, document_id)
+    return jsonify({"result": result}), 200
 
-def retrieve_document(index_id: str, document_id: str) -> Response:
-    """Route to retrieve a document by its ID."""
-    document = current_app.elastic_query_conn.get_document_by_id(  # type: ignore[attr-defined]
-        index_id, document_id
-    )
-    return jsonify(document)
-
-
-def list_wet_processes() -> Response:
-    """Route to list wet processes."""
-    wet_processes_index = (
-        f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-wet_processes"
-    )
-    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
-        wet_processes_index, "proc_id"
-    )
-    return jsonify(list(result))
 
+def wet_processes() -> tuple[Response, int]:
+    """List wet processes."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
 
-def list_bi_processes() -> Response:
-    """Route to list bi processes."""
-    bi_processes_index = (
-        f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-bi_processes"
+    result = es_query_conn.get_field_values(
+        es_query_conn.wet_processes_index,
+        "proc_id",
     )
-    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
-        bi_processes_index, "proc_id"
-    )
-    return jsonify(list(result))
+    return jsonify({"result": list(result)}), 200
 
 
-def list_analyses() -> Response:
-    """Route to list analyses."""
-    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
-    result = current_app.elastic_query_conn.get_field_values(  # type: ignore[attr-defined]
-        analyses_index, "path"
+def get_wet_process(proc_id: str) -> tuple[Response, int]:
+    """Retrieve details of a specific wet process by its ID."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.get_process(
+        es_query_conn.wet_processes_index, proc_id
     )
-    filenames = [Path(path).name for path in result]
-    return jsonify(filenames)
-
-
-def list_analyses_wet_processes(proc_id: str) -> Response:
-    """Route to list analyses one of specific wet process"""
-    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
-
-    search_query = {
-        "query": {
-            "term": {
-                "metadata.wet_process.keyword": proc_id,
-            }
-        }
-    }
-    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-        index=analyses_index, body=search_query
-    )
-    result = [hit["_source"]["path"] for hit in response["hits"]["hits"]]
-
-    return jsonify(result)
-
-
-def list_analyses_bi_processes(proc_id: str) -> Response:
-    """Route to list analyses one of specific bi process"""
-    analyses_index = f"{current_app.config['GENAPI_ES_INDEX_PREFIX']}-analyses"
-
-    search_query = {
-        "query": {
-            "term": {
-                "metadata.bi_process.keyword": proc_id,
-            }
-        }
-    }
-    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-        index=analyses_index, body=search_query
-    )
-    result = [hit["_source"]["path"] for hit in response["hits"]["hits"]]
-
-    return jsonify(result)
-
 
-def list_snv_documents() -> Response:
-    """Route to list all documents containing a mutation at a single position (SNV)."""
-    index_pattern = "genelastic-file-*"
-    target_value = "SNV"
-
-    search_query = {
-        "aggs": {
-            "snv_docs": {
-                "composite": {
-                    "sources": [
-                        {"alt_value": {"terms": {"field": "alt.keyword"}}}
-                    ],
-                    "size": 1000,
+    if not result:
+        return jsonify(
+            {
+                "error": {
+                    "message": f"Wet process with proc_id "
+                    f"'{proc_id}' not found.",
+                    "type": "NotFoundError",
                 }
             }
-        },
-        "query": {"term": {"alt.keyword": target_value}},
-        "size": 0,
-    }
-
-    all_documents = []
-    buckets = current_app.elastic_query_conn.run_composite_aggregation(  # type: ignore[attr-defined]
-        index_pattern, search_query
-    )
-
-    for bucket in buckets:
-        alt_value = bucket["key"]["alt_value"]
-
-        search_query_docs = {
-            "query": {"term": {"alt.keyword": alt_value}},
-            "size": 1000,
-        }
-
-        response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-            index=index_pattern, body=search_query_docs
-        )
+        ), 404
 
-        all_documents.extend(response["hits"]["hits"])
+    return jsonify({"result": result}), 200
 
-    return jsonify(all_documents)
 
+def list_analyses_by_wet_process(proc_id: str) -> tuple[Response, int]:
+    """List analyses that contain the specified wet process."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.list_analyses_by_process("wet_process", proc_id)
+    return jsonify({"result": result}), 200
 
-def build_snv_search_query(
-    target_alt: str, target_svtype: str
-) -> dict[str, Any]:
-    """Helper function to build the search query for SNV documents with specified alt and SVTYPE."""
-    return {
-        "query": {
-            "bool": {
-                "must": [
-                    {"term": {"alt.keyword": target_alt}},
-                    {"term": {"info.SVTYPE.keyword": target_svtype}},
-                ]
-            }
-        },
-        "size": 1000,
-    }
-
-
-def build_snv_mutation_search_query(
-    target_svtypes: list[str],
-) -> dict[str, Any]:
-    """Helper function to build the search query for SNV mutations with specified SVTYPE values."""
-    return {
-        "query": {
-            "bool": {
-                "must": [
-                    {"term": {"alt.keyword": "SNV"}},
-                    {"terms": {"info.SVTYPE.keyword": target_svtypes}},
-                ]
-            }
-        },
-        "size": 1000,
-    }
 
+def bi_processes() -> tuple[Response, int]:
+    """List bioinformatics processes."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.get_field_values(
+        es_query_conn.bi_processes_index, "proc_id"
+    )
+    return jsonify({"result": list(result)}), 200
 
-def list_snv_insertion_documents() -> Response:
-    """Route to list all documents containing an insertion (INS) at a single position (SNV)."""
-    index_pattern = "genelastic-file-*"
-    search_query = build_snv_search_query(target_alt="SNV", target_svtype="INS")
 
-    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-        index=index_pattern, body=search_query
+def get_bi_process(proc_id: str) -> tuple[Response, int]:
+    """Retrieve details of a specific bioinformatics process by its ID."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.get_process(
+        es_query_conn.bi_processes_index, proc_id
     )
 
-    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+    if not result:
+        return jsonify(
+            {
+                "error": {
+                    "message": f"Bioinformatics process with proc_id "
+                    f"'{proc_id}' not found.",
+                    "type": "NotFoundError",
+                }
+            }
+        ), 404
 
-    return jsonify(all_documents)
+    return jsonify({"result": result}), 200
 
 
-def list_snv_deletion_documents() -> Response:
-    """Route to list all documents containing a deletion (DEL) at a single position (SNV)."""
-    index_pattern = "genelastic-file-*"
-    search_query = build_snv_search_query(target_alt="SNV", target_svtype="DEL")
+def list_analyses_by_bi_process(proc_id: str) -> tuple[Response, int]:
+    """List analyses that contain the specified bioinformatics process"""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.list_analyses_by_process("bi_process", proc_id)
+    return jsonify({"result": result}), 200
 
-    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-        index=index_pattern, body=search_query
-    )
 
-    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+def list_analyses_by_bi_process_esql(
+    proc_id: str,
+) -> tuple[Response, int]:
+    """List analyses that contain the specified bioinformatics process using
+    an ES|QL query.
+    """
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.list_analyses_by_process_esql("bi_process", proc_id)
+    return jsonify({"result": result}), 200
 
-    return jsonify(all_documents)
 
+def list_analyses_by_bi_process_sql(
+    proc_id: str,
+) -> tuple[Response, int]:
+    """List analyses that contain the specified bioinformatics process using
+    an SQL query.
+    """
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.list_analyses_by_process_sql("bi_process", proc_id)
+    return jsonify({"result": result}), 200
 
-def list_snv_mutation_documents() -> Response:
-    """Route to list all documents containing a mutation at a single position (SNV)."""
-    index_pattern = "genelastic-file-*"
-    target_svtypes = ["INS", "DEL"]
 
-    search_query = build_snv_mutation_search_query(
-        target_svtypes=target_svtypes
-    )
+def analyses() -> tuple[Response, int]:
+    """List analyses."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
 
-    response = current_app.elastic_query_conn.client.search(  # type: ignore[attr-defined]
-        index=index_pattern, body=search_query
+    result = es_query_conn.get_field_values(
+        es_query_conn.analyses_index, "analysis_id"
     )
+    return jsonify({"result": list(result)}), 200
+
+
+# TODO(?): All the following functions are commented because it is currently
+# unclear what they do and if they are still needed.
+
+# def list_snv_documents() -> Response:
+#     """Route to list all documents containing a mutation at a single position (SNV)."""
+#     es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+#     index_pattern = "genelastic-file-*"
+#     target_value = "SNV"
+#
+#     search_query = {
+#         "aggs": {
+#             "snv_docs": {
+#                 "composite": {
+#                     "sources": [
+#                         {"alt_value": {"terms": {"field": "alt.keyword"}}}
+#                     ],
+#                     "size": 1000,
+#                 }
+#             }
+#         },
+#         "query": {"term": {"alt.keyword": target_value}},
+#         "size": 0,
+#     }
+#
+#     all_documents = []
+#     buckets = es_query_conn.run_composite_aggregation(
+#         index_pattern, search_query
+#     )
+#
+#     for bucket in buckets:
+#         alt_value = bucket["key"]["alt_value"]
+#
+#         search_query_docs = {
+#             "query": {"term": {"alt.keyword": alt_value}},
+#             "size": 1000,
+#         }
+#
+#         response = es_query_conn.client.search(
+#             index=index_pattern, body=search_query_docs
+#         )
+#
+#         all_documents.extend(response["hits"]["hits"])
+#
+#     return jsonify(all_documents)
+#
+#
+# def build_snv_search_query(
+#     target_alt: str, target_svtype: str
+# ) -> dict[str, Any]:
+#     """Helper function to build the search query for SNV documents with specified alt and SVTYPE."""
+#     return {
+#         "query": {
+#             "bool": {
+#                 "must": [
+#                     {"term": {"alt.keyword": target_alt}},
+#                     {"term": {"info.SVTYPE.keyword": target_svtype}},
+#                 ]
+#             }
+#         },
+#         "size": 1000,
+#     }
+#
+#
+# def build_snv_mutation_search_query(
+#     target_svtypes: list[str],
+# ) -> dict[str, Any]:
+#     """Helper function to build the search query for SNV mutations with specified SVTYPE values."""
+#     return {
+#         "query": {
+#             "bool": {
+#                 "must": [
+#                     {"term": {"alt.keyword": "SNV"}},
+#                     {"terms": {"info.SVTYPE.keyword": target_svtypes}},
+#                 ]
+#             }
+#         },
+#         "size": 1000,
+#     }
+#
+#
+# def list_snv_insertion_documents() -> Response:
+#     """Route to list all documents containing an insertion (INS) at a single position (SNV)."""
+#     es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+#
+#     index_pattern = "genelastic-file-*"
+#     search_query = build_snv_search_query(target_alt="SNV", target_svtype="INS")
+#
+#     response = es_query_conn.client.search(
+#         index=index_pattern, body=search_query
+#     )
+#
+#     all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+#
+#     return jsonify(all_documents)
+#
+#
+# def list_snv_deletion_documents() -> Response:
+#     """Route to list all documents containing a deletion (DEL) at a single position (SNV)."""
+#     es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+#     index_pattern = "genelastic-file-*"
+#     search_query = build_snv_search_query(target_alt="SNV", target_svtype="DEL")
+#
+#     response = es_query_conn.client.search(
+#         index=index_pattern, body=search_query
+#     )
+#
+#     all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+#
+#     return jsonify(all_documents)
+#
+#
+# def list_snv_mutation_documents() -> Response:
+#     """Route to list all documents containing a mutation at a single position (SNV)."""
+#     es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+#     index_pattern = "genelastic-file-*"
+#     target_svtypes = ["INS", "DEL"]
+#
+#     search_query = build_snv_mutation_search_query(
+#         target_svtypes=target_svtypes
+#     )
+#
+#     response = es_query_conn.client.search(
+#         index=index_pattern, body=search_query
+#     )
+#
+#     all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
+#
+#     return jsonify(all_documents)
+
+
+def version() -> tuple[Response, int]:
+    """Obtain genelastic package version used by the server."""
+    top_level_package = __package__.split(".")[0]
+    return jsonify({"result": {"version": get_version(top_level_package)}}), 200
+
+
+def key_value_analyses_index() -> tuple[Response, int]:
+    """Route to return all key-values in an index."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    result = es_query_conn.get_all_documents_kv(es_query_conn.analyses_index)
+    return jsonify({"result": result}), 200
+
+
+def count_key_value_analyses_index() -> tuple[Response, int]:
+    """Route to return the key-value counts for fields in an index."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    fields = [
+        "metadata.barcode",
+        "metadata.reference_genome",
+        "metadata.sample_name",
+        "metadata.source",
+    ]
+
+    result: dict[str, dict[str, int]] = {}
+    for field in fields:
+        result[field] = es_query_conn.get_all_documents_kv_count(
+            es_query_conn.analyses_index, field
+        )
 
-    all_documents = [hit["_source"] for hit in response["hits"]["hits"]]
-
-    return jsonify(all_documents)
+    return jsonify({"result": result}), 200
+
+
+def count_wet_process_fields() -> tuple[Response, int]:
+    """Return key-value counts for each field in wet_process documents."""
+    # BUG: this does not work for numeric values.
+    # error_rate_expected, fragmentation, reads_size will always return an
+    # empty dict.
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    wet_fields = [
+        "metadata.amplification",
+        "metadata.error_rate_expected",
+        "metadata.flowcell_type",
+        "metadata.fragmentation",
+        "metadata.generic_kit",
+        "metadata.input_type",
+        "metadata.library_kit",
+        "metadata.manufacturer",
+        "metadata.reads_size",
+        "metadata.sequencer",
+        "metadata.sequencing_kit",
+        "metadata.sequencing_type",
+        "proc_id",
+        "type",
+    ]
+
+    result: dict[str, dict[str, int]] = {}
+    for field in wet_fields:
+        result[field] = es_query_conn.get_all_documents_kv_count(
+            es_query_conn.wet_processes_index, field
+        )
 
+    return jsonify({"result": result}), 200
+
+
+def count_bi_process_fields() -> tuple[Response, int]:
+    """Return key-value counts for each field in bi_process documents, including steps."""
+    es_query_conn: ElasticQueryConn = current_app.elastic_query_conn  # type: ignore[attr-defined]
+    # Champs de niveau racine
+    bi_fields = [
+        "metadata.name",
+        "metadata.pipeline_version",
+        "metadata.sequencing_type",
+        "proc_id",
+        "type",
+    ]
+
+    # Champs des steps (dans la liste steps[])
+    step_fields = [
+        "metadata.steps.name",
+        "metadata.steps.cmd",
+        "metadata.steps.version",
+        "metadata.steps.output",  # optionnel
+    ]
+
+    result: dict[str, dict[str, int]] = {}
+    for field in bi_fields + step_fields:
+        result[field] = es_query_conn.get_all_documents_kv_count(
+            es_query_conn.bi_processes_index, field
+        )
 
-def get_genelastic_version() -> Response:
-    """Retourne la version du package genelastic."""
-    top_level_package = __package__.split(".")[0]
-    return jsonify({"version": version(top_level_package)})
+    return jsonify({"result": result}), 200