genelastic 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

genelastic/ui/server.py

@@ -0,0 +1,87 @@
+import requests
+from flask import Flask, render_template
+
+app = Flask(__name__)
+app.config.from_object("src.genelastic.ui.settings.Config")
+
+
+@app.route("/")
+def home() -> str:
+    api_url = app.config["GENUI_API_URL"]
+    try:
+        version_reponse = requests.get(f"{api_url}version", timeout=20)
+        version = version_reponse.json().get("version")
+        wet_processes_reponse = requests.get(
+            f"{api_url}wet_processes", timeout=20
+        )
+        wet_processes = wet_processes_reponse.json()
+        bi_processes_reponse = requests.get(
+            f"{api_url}bi_processes", timeout=20
+        )
+        bi_processes = bi_processes_reponse.json()
+        analyses_reponse = requests.get(f"{api_url}analyses", timeout=20)
+        analyses = analyses_reponse.json()
+    except requests.exceptions.RequestException:
+        version = "API not reachable"
+        wet_processes = []
+        bi_processes = []
+        analyses = []
+    return render_template(
+        "home.html",
+        version=version,
+        wet_processes=wet_processes,
+        bi_processes=bi_processes,
+        analyses=analyses,
+    )
+
+
+@app.route("/analyses")
+def show_analyses() -> str:
+    api_url = app.config["GENUI_API_URL"]
+    try:
+        analyses_reponse = requests.get(f"{api_url}analyses", timeout=20)
+        analyses = analyses_reponse.json()
+    except requests.exceptions.RequestException:
+        analyses = ["Error fetching data."]
+
+    return render_template("analyses.html", analyses=analyses)
+
+
+@app.route("/bi_processes")
+def show_bi_processes() -> str:
+    api_url = app.config["GENUI_API_URL"]
+    try:
+        bi_processes_reponse = requests.get(
+            f"{api_url}bi_processes", timeout=20
+        )
+        bi_processes = bi_processes_reponse.json()
+    except requests.exceptions.RequestException:
+        bi_processes = ["Error fetching data."]
+
+    return render_template("bi_processes.html", bi_processes=bi_processes)
+
+
+@app.route("/wet_processes")
+def show_wet_processes() -> str:
+    api_url = app.config["GENUI_API_URL"]
+    try:
+        wet_processes_reponse = requests.get(
+            f"{api_url}wet_processes", timeout=20
+        )
+        wet_processes = wet_processes_reponse.json()
+    except requests.exceptions.RequestException:
+        wet_processes = ["Error fetching data."]
+
+    return render_template("wet_processes.html", wet_processes=wet_processes)
+
+
+@app.route("/version")
+def show_version() -> str:
+    api_url = app.config["GENUI_API_URL"]
+    try:
+        version_reponse = requests.get(f"{api_url}version", timeout=20)
+        version = version_reponse.json().get("version", "Version not found")
+    except requests.exceptions.RequestException:
+        version = "Error fetching version."
+
+    return render_template("version.html", version=version)

genelastic/ui/settings.py

@@ -0,0 +1,11 @@
+from environs import Env
+
+env = Env()
+env.read_env()
+
+
+class Config:
+    """Flask config class."""
+
+    # Charger toutes les variables d'environnement nécessaires
+    GENUI_API_URL = env.url("GENUI_API_URL").geturl()

genelastic-0.7.0.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.2
+Name: genelastic
+Version: 0.7.0
+Summary: Generate and store genetic data into an Elasticsearch database.
+Author: CNRGH
+Author-email: Pierrick ROGER <pierrick.roger@cnrgh.fr>, Maxime BLANCHON <maxime.blanchon@cnrgh.fr>
+License: CeCILL
+Keywords: CNRGH,genelastic,generation,storage,elasticsearch,database
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: elasticsearch
+Requires-Dist: PyVCF3
+Requires-Dist: schema
+Requires-Dist: PyYAML
+Requires-Dist: biophony>=1.2.1
+Requires-Dist: colorlog
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: mypy; extra == "tests"
+Requires-Dist: coverage; extra == "tests"
+Requires-Dist: yamllint; extra == "tests"
+Requires-Dist: types-PyYAML; extra == "tests"
+Requires-Dist: ruff<0.9,>=0.8.1; extra == "tests"
+Requires-Dist: pre-commit<4.1,>=4.0.1; extra == "tests"
+Requires-Dist: types-requests; extra == "tests"
+Requires-Dist: ansible<10.5,>=10.4.0; extra == "tests"
+Requires-Dist: ansible-lint<25,>=24.12.2; extra == "tests"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-autoapi; extra == "docs"
+Requires-Dist: furo; extra == "docs"
+Provides-Extra: api
+Requires-Dist: flask; extra == "api"
+Requires-Dist: elasticsearch; extra == "api"
+Requires-Dist: environs; extra == "api"
+Requires-Dist: connexion[flask,swagger-ui,uvicorn]; extra == "api"
+Provides-Extra: ui
+Requires-Dist: flask; extra == "ui"
+Requires-Dist: requests; extra == "ui"
+Requires-Dist: environs; extra == "ui"
+
+# genelastic
+
+Storing of genetics data into an Elasticsearch database.
+
+## Prerequisites
+
+- `python` >= 3.11
+- `make`
+
+## Installation
+
+To install dependencies, run the following command:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+make install.deps
+```
+
+## Configuration
+
+To start the **API server**, the following environment variables should be defined:
+
+- `GENAPI_ES_URL`: URL of the Elasticsearch server,
+- `GENAPI_ES_ENCODED_API_KEY`: Encoded API key,
+- `GENAPI_ES_INDEX_PREFIX`: Prefix to identify indices of interest,
+- `GENAPI_ES_CERT_FP`: Certificate fingerprint of the Elasticsearch server.
+
+Then, run the following command:
+
+```bash
+make start-api
+```
+
+To start the **UI server**, the following environment variables should be defined:
+
+- `GENUI_API_URL`: URL of the API server.
+
+Then, run the following command:
+
+```bash
+make start-ui
+```
+
+## Developers
+
+This project uses [pre-commit](https://pre-commit.com/) to manage Git hooks scripts. To install project hooks, run:
+
+```bash
+pre-commit install
+```
+
+After that, each commit will succeed only if all hooks (defined in `.pre-commit-config.yaml`) pass.
+
+If necessary (though not recommended),
+you can skip these hooks by using the `--no-verify` / `-n` option when committing:
+
+```bash
+git commit -m "My commit message" --no-verify # This commit will not run installed hooks.
+```

genelastic-0.7.0.dist-info/RECORD

@@ -0,0 +1,40 @@
+genelastic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+genelastic/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+genelastic/api/routes.py,sha256=O_bZGhKex04tWauE1fYRXjNmR2smIA4JZShwcakbJJA,6938
+genelastic/api/server.py,sha256=XkkHpv_28OrfrY6CMDnImBFlDiEQSA8f1BpSSn2LGL4,2248
+genelastic/api/settings.py,sha256=vLfj8ASLqq9f6c3eGg30L_WSBiaOIpnCsk-S-TUZRlk,403
+genelastic/api/extends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+genelastic/api/extends/example.py,sha256=5vOw2ToYWzOj-vYkOLbqoa1MCnXIvKuv7xW6qBQ-eHU,164
+genelastic/common/__init__.py,sha256=GfM4XaiaHsA2v8N0T3VgnXpZsLXjTc1wIPpRi4BlkZ8,937
+genelastic/common/cli.py,sha256=cDbgc2m20Y1NO0CZR1qaTrCttcf0cjFAP885tuPZsH0,1770
+genelastic/common/elastic.py,sha256=0XpECyNMC9RzQN1X7u-0IXmHp_KbSjkmGizArLjxTS0,7260
+genelastic/common/exceptions.py,sha256=l4sQBW91OuzV2R7DhrVzEMDrfsyKM-iVYufp2Cv6rR8,159
+genelastic/common/types.py,sha256=JeQ6XzIemIF1QgvprGA0UxFHyJSKjEkgMaIumOvpbNg,1013
+genelastic/import_data/__init__.py,sha256=Ir_fI0BrdC44JiiQaWm29_SyHbDe2CQsElKHgQxELCI,602
+genelastic/import_data/analyses.py,sha256=yA4DKtXIUR7JTCp7PEndEjI6FOooW-6lR9vkqTZ0Izs,1861
+genelastic/import_data/analysis.py,sha256=MuHYfIWFFwHfCp9tDkWTtVKkg5AasASwrdOUZ177EeM,7799
+genelastic/import_data/bi_process.py,sha256=CBBGx6oO7KYzUypXzpaopGY4J_nAKcDi2UPJe7B63Bs,650
+genelastic/import_data/bi_processes.py,sha256=Kv93NOntycEiB_BldoMxHNfM6sJvgHQvVqrUSSEb_VA,1365
+genelastic/import_data/cli_gen_data.py,sha256=WYvSgiC30pDvhQuGMxtDKookUF9IQpjQP6UMANvvHSU,3012
+genelastic/import_data/cli_import.py,sha256=Ej-EqaBUNoVHGCefzLsmFbKaXNTaOBs8xJsEdRhbbLo,11622
+genelastic/import_data/cli_info.py,sha256=ue0Pf5cJvrmJ_bBZKG2CXkXDsQO3zQ6CIXhtUwpqXhU,7301
+genelastic/import_data/cli_integrity.py,sha256=lOMA-I1iPXiJD4X80Xg91NKO-fa3Su0uSqBUtdMKU4Q,12282
+genelastic/import_data/cli_validate.py,sha256=AcirmmJQWFDrpiPUNTYnpFjSnvD6YyStYJHIbfi1Ly0,1515
+genelastic/import_data/constants.py,sha256=AKYXdDqWkDzvt-laZqbWlN6C9IoQSSqD70mpSWMU6NQ,760
+genelastic/import_data/data_file.py,sha256=P5oe_yCVQQAALetrUs93Kl1XuqoLQS0iMkeEqPKaY7g,2661
+genelastic/import_data/filename_pattern.py,sha256=3QXOF5ZG9vFA8KG7XXucpA7EIN3dqZmmtXnuFbh0bUQ,1709
+genelastic/import_data/import_bundle.py,sha256=7W0hKwoxwGDtWEGV_NQvyesiNx1lCMpSaqKohpTLmtY,5030
+genelastic/import_data/import_bundle_factory.py,sha256=COdMLTTrx_Y4svZAHVJ5Y0rHM4eQj027rFJiQ66u36M,9546
+genelastic/import_data/logger.py,sha256=X12LBoNTmV6iR8vEPyYIMYSUeQ3LpTvDO69tQYDiOuA,1915
+genelastic/import_data/random_bundle.py,sha256=Fk4oudK42pboGI14aaJjh8tTJxSxYNyF_89BBJ532RY,13821
+genelastic/import_data/tags.py,sha256=N6_dGYqQy2QTN6AEzoEXPxRFmtq3GgTfpId-RjBstJY,4195
+genelastic/import_data/wet_process.py,sha256=2SgN1yZFQA8Hb4ZNNeHpUhmeWlJDApqdjc_oq6ZF0jA,694
+genelastic/import_data/wet_processes.py,sha256=mVUfFG0QUboTEoD3c-Hd1Z2_Tvid_yrKJG_XdC4n6oA,1535
+genelastic/ui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+genelastic/ui/server.py,sha256=7QX-54tT7CbpKje34kVVXnktI0wuZJOSLUChxeUGd20,2805
+genelastic/ui/settings.py,sha256=hmgDNMVKk_OkKSBKyV8Eth57ULCTINwzeBCFAH7DdeU,218
+genelastic-0.7.0.dist-info/METADATA,sha256=dj4cFcq7LXEv9v-turnGvHnK2KZ4zXrDxG0YKe11zPU,3127
+genelastic-0.7.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+genelastic-0.7.0.dist-info/entry_points.txt,sha256=KHtfefg-7-v0MZ0jAcHrb6z3V6hq7UdCiYYIAlD6hIg,284
+genelastic-0.7.0.dist-info/top_level.txt,sha256=ra4gCsuKH1d0sXygcnwD_u597ir6bYYxWTS7dkA6vdM,11
+genelastic-0.7.0.dist-info/RECORD,,

{genelastic-0.6.0.dist-info → genelastic-0.7.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

genelastic-0.7.0.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[console_scripts]
+gnl-data = genelastic.import_data.cli_gen_data:main
+gnl-import = genelastic.import_data.cli_import:main
+gnl-info = genelastic.import_data.cli_info:main
+gnl-integrity = genelastic.import_data.cli_integrity:main
+gnl-validate = genelastic.import_data.cli_validate:main

genelastic/common.py

@@ -1,151 +0,0 @@
-"""
-Module: common
-
-This module contains custom types and functions shared by multiple genelastic scripts.
-"""
-import argparse
-import sys
-import typing
-import logging
-
-import elastic_transport
-import elasticsearch
-
-logger = logging.getLogger('genelastic')
-
-AnalysisMetaData: typing.TypeAlias = typing.Dict[str, str | int]
-WetProcessesData: typing.TypeAlias = typing.Dict[str, str | int | float]
-BioInfoProcessData: typing.TypeAlias = typing.Dict[str, str | typing.List[str]]
-BundleDict: typing.TypeAlias = typing.Dict[str, typing.Any]
-
-AnalysisDocument: typing.TypeAlias = typing.Dict[str, str | None | AnalysisMetaData]
-MetadataDocument: typing.TypeAlias = typing.Dict[str, int | str | typing.List[typing.Any | None]]
-ProcessDocument: typing.TypeAlias = (typing.Dict[str, str] |
-                                     WetProcessesData |
-                                     BioInfoProcessData)
-BulkItems: typing.TypeAlias = typing.List[typing.Dict[str, str |
-                                                           MetadataDocument |
-                                                           AnalysisDocument |
-                                                           ProcessDocument]]
-Bucket: typing.TypeAlias = typing.Dict[str, typing.Dict[typing.Any, typing.Any]]
-
-
-def connect_to_es(host: str, port: int, usr: str, pwd: str) -> elasticsearch.Elasticsearch:
-    """Connect to a remote Elasticsearch database."""
-    addr = f"https://{host}:{port}"
-    logger.info("Trying to connect to Elasticsearch at %s.", addr)
-
-    try:
-        es = elasticsearch.Elasticsearch(
-            addr,
-            # ssl_assert_fingerprint=args.es_cert_fp,
-            # ca_certs=args.es_cert,
-            verify_certs=False,
-            basic_auth=(usr, pwd)
-        )
-        logger.info(es.info())
-    except elastic_transport.TransportError as e:
-        logger.error(e.message)
-        sys.exit(1)
-    return es
-
-
-def run_composite_aggregation(es: elasticsearch.Elasticsearch,
-                              index: str, query: typing.Dict[str, typing.Any]) \
-        -> typing.List[Bucket]:
-    """
-    Executes a composite aggregation on an Elasticsearch index and returns all paginated results.
-
-    :param es: Elasticsearch client instance.
-    :param index: Name of the index to query.
-    :param query: Aggregation query to run.
-    :return: List of aggregation results.
-    """
-    # Extract the aggregation name from the query dict.
-    agg_name = next(iter(query["aggs"]))
-    all_buckets: typing.List[Bucket] = []
-
-    try:
-        logger.debug("Running composite aggregation query %s on index '%s'.", query, index)
-        response = es.search(index=index, body=query)
-    except elasticsearch.NotFoundError as e:
-        raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
-
-    while True:
-        # Extract buckets from the response.
-        buckets: typing.List[Bucket] = response['aggregations'][agg_name]['buckets']
-        all_buckets.extend(buckets)
-
-        # Check if there are more results to fetch.
-        if 'after_key' in response['aggregations'][agg_name]:
-            after_key = response['aggregations'][agg_name]['after_key']
-            query['aggs'][agg_name]['composite']['after'] = after_key
-            try:
-                logger.debug("Running query %s on index '%s'.", query, index)
-                response = es.search(index=index, body=query)  # Fetch the next page of results.
-            except elasticsearch.NotFoundError as e:
-                raise SystemExit(f"Error: {e.message} for index '{index}'.") from e
-        else:
-            break
-
-    return all_buckets
-
-
-def get_process_ids(es: elasticsearch.Elasticsearch, index: str, proc_field_name: str) \
-        -> typing.Set[str]:
-    """Return a set of process IDs."""
-    process_ids = set()
-
-    query = {
-        "size": 0,
-        "aggs": {
-            "get_proc_ids": {
-                "composite": {
-                    "sources": {"proc_id": {"terms": {"field": f"{proc_field_name}.keyword"}}},
-                    "size": 1000,
-                }
-            }
-        }
-    }
-
-    buckets: typing.List[Bucket] = run_composite_aggregation(es, index, query)
-
-    for bucket in buckets:
-        process_ids.add(bucket['key']['proc_id'])
-
-    return process_ids
-
-
-def add_verbose_control_args(parser: argparse.ArgumentParser) -> None:
-    """
-    Add verbose control arguments to the parser.
-    Arguments are added to the parser by using its reference.
-    """
-    parser.add_argument('-q', '--quiet', dest='verbose', action='store_const',
-                        const=0, default=1,
-                        help='Set verbosity to 0 (quiet mode).')
-    parser.add_argument('-v', '--verbose', dest='verbose', action='count',
-                        default=1,
-                        help=('Verbose level. -v for information, -vv for debug,' +
-                              ' -vvv for trace.'))
-
-
-def add_es_connection_args(parser: argparse.ArgumentParser) -> None:
-    """
-    Add arguments to the parser needed to gather ElasticSearch server connection parameters.
-    Arguments are added to the parser by using its reference.
-    """
-    parser.add_argument('--es-host', dest='es_host', default='localhost',
-                        help='Address of Elasticsearch host.')
-    parser.add_argument('--es-port', type=int, default=9200, dest='es_port',
-                        help='Elasticsearch port.')
-    parser.add_argument('--es-usr', dest='es_usr', default='elastic',
-                        help='Elasticsearch user.')
-    parser.add_argument('--es-pwd', dest='es_pwd', required=True,
-                        help='Elasticsearch password.')
-    parser.add_argument('--es-cert', dest='es_cert',
-                        help='Elasticsearch certificate file.')
-    parser.add_argument('--es-cert-fp', dest='es_cert_fp',
-                        help='Elasticsearch certificate fingerprint.')
-    parser.add_argument('--es-index-prefix', dest='es_index_prefix',
-                        help='Add the given prefix to each index created during import.')

genelastic/constants.py

@@ -1,45 +0,0 @@
-"""
-Module: constants
-
-This module contains genelastic constants.
-"""
-import typing
-
-ALLOWED_CATEGORIES: typing.Final[typing.List[str]] = ['vcf', 'cov']
-
-BUNDLE_CURRENT_VERSION = 3
-
-DEFAULT_TAG_REGEX = "[^_-]+"
-DEFAULT_TAG_PREFIX = "%"
-DEFAULT_TAG_SUFFIX = ""
-
-DEFAULT_TAG2FIELD: typing.Final[typing.Dict[str, typing.Dict[str, str]]] = {
-    '%S': {
-        "field": 'sample_name',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%F': {
-        "field": 'source',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%W': {
-        "field": 'wet_process',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%B': {
-        "field": 'bi_process',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%D': {
-        "field": 'cov_depth',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%A': {
-        "field": 'barcode',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%R':  {
-        "field": 'reference_genome',
-        "regex": DEFAULT_TAG_REGEX
-    }
-}

genelastic/filename_pattern.py

@@ -1,62 +0,0 @@
-"""
-This module defines the FilenamePattern class, used to define a filename pattern
-and extract metadata from file names using this pattern.
-"""
-
-import logging
-import re
-from .common import AnalysisMetaData
-
-logger = logging.getLogger('genelastic')
-
-class FilenamePattern:
-    """Class for defining a filename pattern.
-    The pattern is used to extract metadata from filenames
-    and verify filename conformity.
-    """
-
-    # Initializer
-    def __init__(self, pattern: str) -> None:
-        """
-                Initializes a FilenamePattern instance.
-
-                Args:
-                    pattern (str): The pattern string used for defining
-                     the filename pattern.
-                """
-        self._re = re.compile(pattern)
-
-    def extract_metadata(self, filename: str) -> AnalysisMetaData:
-        """
-                Extracts metadata from the given filename based
-                on the defined pattern.
-
-                Args:
-                    filename (str): The filename from which metadata
-                    needs to be extracted.
-
-                Returns:
-                    dict: A dictionary containing the extracted metadata.
-
-                Raises:
-                    RuntimeError: If parsing of filename fails
-                    with the defined pattern.
-                """
-        m = self._re.search(filename)
-        if not m:
-            raise RuntimeError(f'Failed parsing filename "{filename}"' +
-                               f'with pattern "{self._re.pattern}".')
-        return m.groupdict()
-
-    def matches_pattern(self, filename: str) -> bool:
-        """
-                Checks if the given filename matches the defined pattern.
-
-                Args:
-                    filename (str): The filename to be checked.
-
-                Returns:
-                    bool: True if the filename matches the pattern,
-                    False otherwise.
-                """
-        return bool(self._re.match(filename))