genelastic 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {genelastic-0.9.0 → genelastic-0.10.0}/PKG-INFO +1 -1
- {genelastic-0.9.0 → genelastic-0.10.0}/pyproject.toml +1 -1
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/cli.py +4 -11
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/cli_check.py +3 -4
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/import_data.py +2 -3
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/info.py +2 -3
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/integrity.py +2 -3
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_base.py +6 -1
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_types.py +13 -2
- {genelastic-0.9.0 → genelastic-0.10.0}/LICENSE +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/README.md +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/.env +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/cli_start_api.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/errors.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/extends/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/extends/example.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/extends/example.yml +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/routes.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/server.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/settings.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/api/specification.yml +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/elastic.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/exceptions.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/server.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/common/types.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/checker.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/checker_observer.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/gen_data.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/cli/validate.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/collect.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/constants.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/import_bundle.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/import_bundle_factory.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_factory.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/logger.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/analyses.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/analysis.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/data_file.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/process.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/processes.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/tags.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/unique_list.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/models/validate.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/patterns.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/random_bundle.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/resolve.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/.env +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/__init__.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/cli_start_ui.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/routes.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/server.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/settings.py +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/cea-cnrgh.ico +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/cea.ico +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/layout.ico +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/novaseq6000.png +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/style.css +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/static/ui.js +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/analyses.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/analysis_detail.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/bi_process_detail.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/bi_processes.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/explorer.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/home.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/layout.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/version.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/wet_process_detail.html +0 -0
- {genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/ui/templates/wet_processes.html +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: genelastic
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: Generate and store genetic data into an Elasticsearch database.
|
|
5
5
|
Keywords: CNRGH,genelastic,generation,storage,elasticsearch,database
|
|
6
6
|
Author: CNRGH, Pierrick ROGER, Maxime BLANCHON
|
|
@@ -66,17 +66,10 @@ def add_es_connection_args(parser: argparse.ArgumentParser) -> None:
|
|
|
66
66
|
Arguments are added to the parser by using its reference.
|
|
67
67
|
"""
|
|
68
68
|
parser.add_argument(
|
|
69
|
-
"--es-
|
|
70
|
-
dest="
|
|
71
|
-
default="localhost",
|
|
72
|
-
help="
|
|
73
|
-
)
|
|
74
|
-
parser.add_argument(
|
|
75
|
-
"--es-port",
|
|
76
|
-
type=int,
|
|
77
|
-
default=9200,
|
|
78
|
-
dest="es_port",
|
|
79
|
-
help="Elasticsearch port.",
|
|
69
|
+
"--es-url",
|
|
70
|
+
dest="es_url",
|
|
71
|
+
default="https://localhost:9200/",
|
|
72
|
+
help="URL of the Elasticsearch host.",
|
|
80
73
|
)
|
|
81
74
|
parser.add_argument(
|
|
82
75
|
"--es-usr", dest="es_usr", default="elastic", help="Elasticsearch user."
|
|
@@ -96,13 +96,12 @@ def main() -> None:
|
|
|
96
96
|
configure_logging(args.verbose)
|
|
97
97
|
|
|
98
98
|
logger.info(
|
|
99
|
-
"Connecting to Elasticsearch at
|
|
100
|
-
args.
|
|
101
|
-
args.es_port,
|
|
99
|
+
"Connecting to Elasticsearch at %s...",
|
|
100
|
+
args.es_url,
|
|
102
101
|
)
|
|
103
102
|
|
|
104
103
|
es = ElasticQueryConn(
|
|
105
|
-
|
|
104
|
+
args.es_url,
|
|
106
105
|
args.es_cert_fp,
|
|
107
106
|
args.es_index_prefix,
|
|
108
107
|
basic_auth=(args.es_usr, args.es_pwd),
|
|
@@ -234,10 +234,9 @@ def main() -> None:
|
|
|
234
234
|
logger.debug("LOGGERS: %s", logging.root.manager.loggerDict)
|
|
235
235
|
|
|
236
236
|
# Open connection to ES
|
|
237
|
-
|
|
238
|
-
logger.info("Connecting to Elasticsearch at %s...", addr)
|
|
237
|
+
logger.info("Connecting to Elasticsearch at %s...", args.es_url)
|
|
239
238
|
es_import_conn = ElasticImportConn(
|
|
240
|
-
|
|
239
|
+
args.es_url,
|
|
241
240
|
args.es_cert_fp,
|
|
242
241
|
args.es_index_prefix,
|
|
243
242
|
args.dryrun,
|
|
@@ -209,10 +209,9 @@ def main() -> None:
|
|
|
209
209
|
configure_logging(args.verbose)
|
|
210
210
|
logger.debug("Arguments: %s", args)
|
|
211
211
|
|
|
212
|
-
|
|
213
|
-
logger.info("Connecting to Elasticsearch at %s...", addr)
|
|
212
|
+
logger.info("Connecting to Elasticsearch at %s...", args.es_url)
|
|
214
213
|
es_query_conn = ElasticQueryConn(
|
|
215
|
-
|
|
214
|
+
args.es_url,
|
|
216
215
|
args.es_cert_fp,
|
|
217
216
|
args.es_index_prefix,
|
|
218
217
|
basic_auth=(args.es_usr, args.es_pwd),
|
|
@@ -366,10 +366,9 @@ def main() -> None:
|
|
|
366
366
|
wet_processes_index = f"{args.es_index_prefix}-wet_processes"
|
|
367
367
|
bi_processes_index = f"{args.es_index_prefix}-bi_processes"
|
|
368
368
|
|
|
369
|
-
|
|
370
|
-
logger.info("Connecting to Elasticsearch at %s...", addr)
|
|
369
|
+
logger.info("Connecting to Elasticsearch at %s...", args.es_url)
|
|
371
370
|
es_query_conn = ElasticQueryConn(
|
|
372
|
-
|
|
371
|
+
args.es_url,
|
|
373
372
|
args.es_cert_fp,
|
|
374
373
|
args.es_index_prefix,
|
|
375
374
|
args.dryrun,
|
{genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_base.py
RENAMED
|
@@ -72,7 +72,12 @@ class BaseImporter(ABC, Generic[T]):
|
|
|
72
72
|
"""Load and parse raw data from the file."""
|
|
73
73
|
|
|
74
74
|
def _validate(self, data: T) -> None:
|
|
75
|
-
"""Validate the data structure (optional).
|
|
75
|
+
"""Validate the data structure (optional).
|
|
76
|
+
|
|
77
|
+
Note: Do not implement for streaming-based importers (generators).
|
|
78
|
+
Generators should not be consumed here; validate per-record in
|
|
79
|
+
``_transform`` if needed.
|
|
80
|
+
"""
|
|
76
81
|
|
|
77
82
|
@abstractmethod
|
|
78
83
|
def _transform(self, data: T) -> Iterable[dict[str, Any]]:
|
{genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_types.py
RENAMED
|
@@ -77,15 +77,26 @@ class VCFImporter(BaseImporter[Iterable[_Record]]):
|
|
|
77
77
|
:raises ImporterError: If the file could not be opened, decoded or is empty.
|
|
78
78
|
"""
|
|
79
79
|
try:
|
|
80
|
-
|
|
80
|
+
# StopIteration can come from two sources:
|
|
81
|
+
# 1. vcf.Reader() constructor if file is empty
|
|
82
|
+
# (unhandled in header parsing)
|
|
83
|
+
# 2. next() if file has valid headers but no variant records
|
|
84
|
+
reader = vcf.Reader(
|
|
81
85
|
filename=str(self._data_file.path), encoding="utf-8"
|
|
82
86
|
)
|
|
87
|
+
first_record = next(reader)
|
|
83
88
|
except StopIteration:
|
|
84
|
-
msg =
|
|
89
|
+
msg = (
|
|
90
|
+
f"VCF file '{self._data_file.path}' is empty or "
|
|
91
|
+
f"contains no variant records."
|
|
92
|
+
)
|
|
85
93
|
raise ImporterError(msg) from None
|
|
86
94
|
except (OSError, UnicodeDecodeError) as e:
|
|
87
95
|
raise ImporterError(e) from None
|
|
88
96
|
|
|
97
|
+
yield first_record
|
|
98
|
+
yield from reader
|
|
99
|
+
|
|
89
100
|
def _transform(self, data: Iterable[_Record]) -> Iterable[dict[str, Any]]:
|
|
90
101
|
"""Transform each VCF file record into a JSON document."""
|
|
91
102
|
for record in data:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{genelastic-0.9.0 → genelastic-0.10.0}/src/genelastic/import_data/importers/importer_factory.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|