looper 1.5.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/__init__.py +3 -498
- looper/__main__.py +2 -2
- looper/_version.py +1 -1
- looper/cli_divvy.py +182 -0
- looper/cli_looper.py +776 -0
- looper/conductor.py +53 -206
- looper/const.py +51 -3
- looper/divvy.py +28 -196
- looper/exceptions.py +18 -0
- looper/looper.py +177 -612
- looper/plugins.py +160 -0
- looper/processed_project.py +1 -1
- looper/project.py +229 -117
- looper/utils.py +119 -43
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/METADATA +6 -6
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/RECORD +20 -20
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/WHEEL +1 -1
- looper/html_reports.py +0 -1057
- looper/html_reports_pipestat.py +0 -924
- looper/html_reports_project_pipestat.py +0 -269
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/LICENSE.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/entry_points.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/top_level.txt +0 -0
looper/html_reports_pipestat.py
DELETED
@@ -1,924 +0,0 @@
|
|
1
|
-
""" Generate HTML reports """
|
2
|
-
|
3
|
-
import logging
|
4
|
-
import os
|
5
|
-
import sys
|
6
|
-
from datetime import timedelta
|
7
|
-
from json import dumps
|
8
|
-
|
9
|
-
import jinja2
|
10
|
-
import pandas as _pd
|
11
|
-
from eido import read_schema
|
12
|
-
from peppy.const import *
|
13
|
-
|
14
|
-
from ._version import __version__ as v
|
15
|
-
from .const import *
|
16
|
-
from .utils import get_file_for_project
|
17
|
-
|
18
|
-
_LOGGER = logging.getLogger("looper")
|
19
|
-
|
20
|
-
|
21
|
-
class HTMLReportBuilder(object):
|
22
|
-
"""Generate HTML summary report for project/samples"""
|
23
|
-
|
24
|
-
def __init__(self, prj):
|
25
|
-
"""
|
26
|
-
The Project defines the instance.
|
27
|
-
|
28
|
-
:param looper.Project prj: Project with which to work/operate on
|
29
|
-
"""
|
30
|
-
super(HTMLReportBuilder, self).__init__()
|
31
|
-
self.prj = prj
|
32
|
-
self.j_env = get_jinja_env()
|
33
|
-
self.output_dir = self.prj.output_dir
|
34
|
-
self.reports_dir = os.path.join(self.output_dir, "reports")
|
35
|
-
_LOGGER.debug(f"Reports dir: {self.reports_dir}")
|
36
|
-
|
37
|
-
def __call__(self, pipeline_name, project_index_html=None):
|
38
|
-
"""
|
39
|
-
Generate HTML report.
|
40
|
-
|
41
|
-
:param str pipeline_name: ID of the pipeline to generate the report for
|
42
|
-
:return str: path to the index page of the generated HTML report
|
43
|
-
"""
|
44
|
-
# Generate HTML report
|
45
|
-
self.pipeline_name = pipeline_name
|
46
|
-
self.amendments_str = (
|
47
|
-
"_".join(self.prj.amendments) if self.prj.amendments else ""
|
48
|
-
)
|
49
|
-
self.pipeline_reports = os.path.join(
|
50
|
-
self.reports_dir,
|
51
|
-
f"{self.pipeline_name}_{self.amendments_str}"
|
52
|
-
if self.prj.amendments
|
53
|
-
else self.pipeline_name,
|
54
|
-
)
|
55
|
-
self.prj_index_html_path = project_index_html
|
56
|
-
self.index_html_path = os.path.join(self.pipeline_reports, "index.html")
|
57
|
-
pifaces = self.prj.pipeline_interfaces
|
58
|
-
selected_pipeline_pifaces = [
|
59
|
-
p for p in pifaces if p.pipeline_name == self.pipeline_name
|
60
|
-
]
|
61
|
-
schema_path = self.prj.get_schemas(
|
62
|
-
selected_pipeline_pifaces, OUTPUT_SCHEMA_KEY
|
63
|
-
)[0]
|
64
|
-
self.schema = read_schema(schema_path)[0]
|
65
|
-
navbar = self.create_navbar(
|
66
|
-
navbar_links=self.create_navbar_links(
|
67
|
-
wd=self.pipeline_reports,
|
68
|
-
project_index_html_relpath=os.path.relpath(
|
69
|
-
self.prj_index_html_path, self.pipeline_reports
|
70
|
-
)
|
71
|
-
if self.prj_index_html_path
|
72
|
-
else None,
|
73
|
-
),
|
74
|
-
index_html_relpath=os.path.relpath(
|
75
|
-
self.index_html_path, self.pipeline_reports
|
76
|
-
),
|
77
|
-
)
|
78
|
-
self.create_index_html(navbar, self.create_footer())
|
79
|
-
return self.index_html_path
|
80
|
-
|
81
|
-
def create_object_parent_html(self, navbar, footer):
|
82
|
-
"""
|
83
|
-
Generates a page listing all the project objects with links
|
84
|
-
to individual object pages
|
85
|
-
|
86
|
-
:param str navbar: HTML to be included as the navbar in the main summary page
|
87
|
-
:param str footer: HTML to be included as the footer
|
88
|
-
:return str: Rendered parent objects HTML file
|
89
|
-
"""
|
90
|
-
if not os.path.exists(self.pipeline_reports):
|
91
|
-
os.makedirs(self.pipeline_reports)
|
92
|
-
pages = list()
|
93
|
-
labels = list()
|
94
|
-
obj_result_ids = self.get_nonhighlighted_results(OBJECT_TYPES)
|
95
|
-
|
96
|
-
for key in obj_result_ids:
|
97
|
-
desc = (
|
98
|
-
self.schema[key]["description"]
|
99
|
-
if "description" in self.schema[key]
|
100
|
-
else ""
|
101
|
-
)
|
102
|
-
labels.append(f"<b>{key.replace('_', ' ')}</b>: {desc}")
|
103
|
-
page_path = os.path.join(self.pipeline_reports, f"{key}.html".lower())
|
104
|
-
pages.append(os.path.relpath(page_path, self.pipeline_reports))
|
105
|
-
|
106
|
-
template_vars = dict(
|
107
|
-
navbar=navbar, footer=footer, labels=labels, pages=pages, header="Objects"
|
108
|
-
)
|
109
|
-
_LOGGER.debug(
|
110
|
-
f"object navbar_list_parent.html | template_vars:" f"\n{template_vars}"
|
111
|
-
)
|
112
|
-
return render_jinja_template(
|
113
|
-
"navbar_list_parent.html", self.j_env, template_vars
|
114
|
-
)
|
115
|
-
|
116
|
-
def create_sample_parent_html(self, navbar, footer):
|
117
|
-
"""
|
118
|
-
Generates a page listing all the project samples with links
|
119
|
-
to individual sample pages
|
120
|
-
:param str navbar: HTML to be included as the navbar in the main summary page
|
121
|
-
:param str footer: HTML to be included as the footer
|
122
|
-
:return str: Rendered parent samples HTML file
|
123
|
-
"""
|
124
|
-
if not os.path.exists(self.pipeline_reports):
|
125
|
-
os.makedirs(self.pipeline_reports)
|
126
|
-
pages = list()
|
127
|
-
labels = list()
|
128
|
-
for sample in self.prj.samples:
|
129
|
-
sample_name = str(sample.sample_name)
|
130
|
-
sample_dir = os.path.join(self.prj.results_folder, sample_name)
|
131
|
-
|
132
|
-
# Confirm sample directory exists, then build page
|
133
|
-
if os.path.exists(sample_dir):
|
134
|
-
page_path = os.path.join(
|
135
|
-
self.pipeline_reports,
|
136
|
-
f"{sample_name}.html".replace(" ", "_").lower(),
|
137
|
-
)
|
138
|
-
page_relpath = os.path.relpath(page_path, self.pipeline_reports)
|
139
|
-
pages.append(page_relpath)
|
140
|
-
labels.append(sample_name)
|
141
|
-
|
142
|
-
template_vars = dict(
|
143
|
-
navbar=navbar, footer=footer, labels=labels, pages=pages, header="Samples"
|
144
|
-
)
|
145
|
-
_LOGGER.debug(
|
146
|
-
f"sample navbar_list_parent.html | template_vars:" f"\n{template_vars}"
|
147
|
-
)
|
148
|
-
return render_jinja_template(
|
149
|
-
"navbar_list_parent.html", self.j_env, template_vars
|
150
|
-
)
|
151
|
-
|
152
|
-
def create_navbar(self, navbar_links, index_html_relpath):
|
153
|
-
"""
|
154
|
-
Creates the navbar using the provided links
|
155
|
-
|
156
|
-
:param str navbar_links: HTML list of links to be inserted into a navbar
|
157
|
-
:return str: navbar HTML
|
158
|
-
"""
|
159
|
-
template_vars = dict(navbar_links=navbar_links, index_html=index_html_relpath)
|
160
|
-
return render_jinja_template("navbar.html", self.j_env, template_vars)
|
161
|
-
|
162
|
-
def create_footer(self):
|
163
|
-
"""
|
164
|
-
Renders the footer from the templates directory
|
165
|
-
|
166
|
-
:return str: footer HTML
|
167
|
-
"""
|
168
|
-
return render_jinja_template("footer.html", self.j_env, dict(version=v))
|
169
|
-
|
170
|
-
def create_navbar_links(
|
171
|
-
self, wd=None, context=None, project_index_html_relpath=None
|
172
|
-
):
|
173
|
-
"""
|
174
|
-
Return a string containing the navbar prebuilt html.
|
175
|
-
|
176
|
-
Generates links to each page relative to the directory of interest
|
177
|
-
(wd arg) or uses the provided context to create the paths (context arg)
|
178
|
-
|
179
|
-
:param path wd: the working directory of the current HTML page being
|
180
|
-
generated, enables navbar links relative to page
|
181
|
-
:param list[str] context: the context the links will be used in.
|
182
|
-
The sequence of directories to be prepended to the HTML file in
|
183
|
-
the resulting navbar
|
184
|
-
:return str: navbar links as HTML-formatted string
|
185
|
-
"""
|
186
|
-
# determine paths
|
187
|
-
if wd is None and context is None:
|
188
|
-
raise ValueError(
|
189
|
-
"Either 'wd' (path the links should be relative to) or "
|
190
|
-
"'context' (the context for the links) has to be provided."
|
191
|
-
)
|
192
|
-
status_relpath = _make_relpath(
|
193
|
-
file_name=os.path.join(self.pipeline_reports, "status.html"),
|
194
|
-
wd=wd,
|
195
|
-
context=context,
|
196
|
-
)
|
197
|
-
objects_relpath = _make_relpath(
|
198
|
-
file_name=os.path.join(self.pipeline_reports, "objects.html"),
|
199
|
-
wd=wd,
|
200
|
-
context=context,
|
201
|
-
)
|
202
|
-
samples_relpath = _make_relpath(
|
203
|
-
file_name=os.path.join(self.pipeline_reports, "samples.html"),
|
204
|
-
wd=wd,
|
205
|
-
context=context,
|
206
|
-
)
|
207
|
-
# determine the outputs IDs by type
|
208
|
-
obj_result_ids = self.get_nonhighlighted_results(OBJECT_TYPES)
|
209
|
-
dropdown_keys_objects = None
|
210
|
-
dropdown_relpaths_objects = None
|
211
|
-
sample_names = None
|
212
|
-
if len(obj_result_ids) > 0:
|
213
|
-
# If the number of objects is 20 or less, use a drop-down menu
|
214
|
-
if len(obj_result_ids) <= 20:
|
215
|
-
(
|
216
|
-
dropdown_relpaths_objects,
|
217
|
-
dropdown_keys_objects,
|
218
|
-
) = self._get_navbar_dropdown_data_objects(
|
219
|
-
objs=obj_result_ids, wd=wd, context=context
|
220
|
-
)
|
221
|
-
else:
|
222
|
-
dropdown_relpaths_objects = objects_relpath
|
223
|
-
if len(self.prj.samples) <= 20:
|
224
|
-
(
|
225
|
-
dropdown_relpaths_samples,
|
226
|
-
sample_names,
|
227
|
-
) = self._get_navbar_dropdown_data_samples(wd=wd, context=context)
|
228
|
-
else:
|
229
|
-
# Create a menu link to the samples parent page
|
230
|
-
dropdown_relpaths_samples = samples_relpath
|
231
|
-
template_vars = dict(
|
232
|
-
status_html_page=status_relpath,
|
233
|
-
status_page_name="Status",
|
234
|
-
dropdown_keys_objects=dropdown_keys_objects,
|
235
|
-
objects_page_name="Objects",
|
236
|
-
samples_page_name="Samples",
|
237
|
-
objects_html_page=dropdown_relpaths_objects,
|
238
|
-
samples_html_page=dropdown_relpaths_samples,
|
239
|
-
menu_name_objects="Objects",
|
240
|
-
menu_name_samples="Samples",
|
241
|
-
sample_names=sample_names,
|
242
|
-
all_samples=samples_relpath,
|
243
|
-
all_objects=objects_relpath,
|
244
|
-
sample_reports_parent=None,
|
245
|
-
project_report=project_index_html_relpath,
|
246
|
-
)
|
247
|
-
_LOGGER.debug(f"navbar_links.html | template_vars:\n{template_vars}")
|
248
|
-
return render_jinja_template("navbar_links.html", self.j_env, template_vars)
|
249
|
-
|
250
|
-
def create_object_htmls(self, navbar, footer):
|
251
|
-
"""
|
252
|
-
Generates a page for an individual object type with all of its
|
253
|
-
plots from each sample
|
254
|
-
|
255
|
-
:param str navbar: HTML to be included as the navbar in the main summary page
|
256
|
-
:param str footer: HTML to be included as the footer
|
257
|
-
"""
|
258
|
-
file_results = self.get_nonhighlighted_results(["file"])
|
259
|
-
image_results = self.get_nonhighlighted_results(["image"])
|
260
|
-
|
261
|
-
if not os.path.exists(self.pipeline_reports):
|
262
|
-
os.makedirs(self.pipeline_reports)
|
263
|
-
for file_result in file_results:
|
264
|
-
links = []
|
265
|
-
html_page_path = os.path.join(
|
266
|
-
self.pipeline_reports, f"{file_result}.html".lower()
|
267
|
-
)
|
268
|
-
for sample in self.prj.samples:
|
269
|
-
sample_result = fetch_pipeline_results(
|
270
|
-
project=self.prj,
|
271
|
-
pipeline_name=self.pipeline_name,
|
272
|
-
sample_name=sample.sample_name,
|
273
|
-
)
|
274
|
-
if file_result not in sample_result:
|
275
|
-
break
|
276
|
-
sample_result = sample_result[file_result]
|
277
|
-
links.append(
|
278
|
-
[
|
279
|
-
sample.sample_name,
|
280
|
-
os.path.relpath(sample_result["path"], self.pipeline_reports),
|
281
|
-
]
|
282
|
-
)
|
283
|
-
else:
|
284
|
-
link_desc = (
|
285
|
-
self.schema[file_result]["description"]
|
286
|
-
if "description" in self.schema[file_result]
|
287
|
-
else "No description in schema"
|
288
|
-
)
|
289
|
-
template_vars = dict(
|
290
|
-
navbar=navbar,
|
291
|
-
footer=footer,
|
292
|
-
name=sample_result["title"],
|
293
|
-
figures=[],
|
294
|
-
links=links,
|
295
|
-
desc=link_desc,
|
296
|
-
)
|
297
|
-
save_html(
|
298
|
-
html_page_path,
|
299
|
-
render_jinja_template(
|
300
|
-
"object.html", self.j_env, args=template_vars
|
301
|
-
),
|
302
|
-
)
|
303
|
-
|
304
|
-
for image_result in image_results:
|
305
|
-
html_page_path = os.path.join(
|
306
|
-
self.pipeline_reports, f"{image_result}.html".lower()
|
307
|
-
)
|
308
|
-
figures = []
|
309
|
-
for sample in self.prj.samples:
|
310
|
-
sample_result = fetch_pipeline_results(
|
311
|
-
project=self.prj,
|
312
|
-
pipeline_name=self.pipeline_name,
|
313
|
-
sample_name=sample.sample_name,
|
314
|
-
)
|
315
|
-
if image_result not in sample_result:
|
316
|
-
break
|
317
|
-
sample_result = sample_result[image_result]
|
318
|
-
figures.append(
|
319
|
-
[
|
320
|
-
os.path.relpath(sample_result["path"], self.pipeline_reports),
|
321
|
-
sample.sample_name,
|
322
|
-
os.path.relpath(
|
323
|
-
sample_result["thumbnail_path"], self.pipeline_reports
|
324
|
-
),
|
325
|
-
]
|
326
|
-
)
|
327
|
-
else:
|
328
|
-
img_desc = (
|
329
|
-
self.schema[image_result]["description"]
|
330
|
-
if "description" in self.schema[image_result]
|
331
|
-
else "No description in schema"
|
332
|
-
)
|
333
|
-
template_vars = dict(
|
334
|
-
navbar=navbar,
|
335
|
-
footer=footer,
|
336
|
-
name=sample_result["title"],
|
337
|
-
figures=figures,
|
338
|
-
links=[],
|
339
|
-
desc=img_desc,
|
340
|
-
)
|
341
|
-
_LOGGER.debug(f"object.html | template_vars:\n{template_vars}")
|
342
|
-
save_html(
|
343
|
-
html_page_path,
|
344
|
-
render_jinja_template(
|
345
|
-
"object.html", self.j_env, args=template_vars
|
346
|
-
),
|
347
|
-
)
|
348
|
-
|
349
|
-
def create_sample_html(self, sample_stats, navbar, footer, sample_name):
|
350
|
-
"""
|
351
|
-
Produce an HTML page containing all of a sample's objects
|
352
|
-
and the sample summary statistics
|
353
|
-
|
354
|
-
:param str sample_name: the name of the current sample
|
355
|
-
:param dict sample_stats: pipeline run statistics for the current sample
|
356
|
-
:param str navbar: HTML to be included as the navbar in the main summary page
|
357
|
-
:param str footer: HTML to be included as the footer
|
358
|
-
:return str: path to the produced HTML page
|
359
|
-
"""
|
360
|
-
if not os.path.exists(self.pipeline_reports):
|
361
|
-
os.makedirs(self.pipeline_reports)
|
362
|
-
html_page = os.path.join(self.pipeline_reports, f"{sample_name}.html".lower())
|
363
|
-
|
364
|
-
psms = self.prj.get_pipestat_managers(sample_name=sample_name)
|
365
|
-
psm = psms[self.pipeline_name]
|
366
|
-
flag = psm.get_status()
|
367
|
-
if not flag:
|
368
|
-
button_class = "btn btn-secondary"
|
369
|
-
flag = "Missing"
|
370
|
-
else:
|
371
|
-
try:
|
372
|
-
flag_dict = BUTTON_APPEARANCE_BY_FLAG[flag]
|
373
|
-
except KeyError:
|
374
|
-
button_class = "btn btn-secondary"
|
375
|
-
flag = "Unknown"
|
376
|
-
else:
|
377
|
-
button_class = flag_dict["button_class"]
|
378
|
-
flag = flag_dict["flag"]
|
379
|
-
highlighted_results = fetch_pipeline_results(
|
380
|
-
project=self.prj,
|
381
|
-
pipeline_name=self.pipeline_name,
|
382
|
-
sample_name=sample_name,
|
383
|
-
inclusion_fun=lambda x: x == "file",
|
384
|
-
highlighted=True,
|
385
|
-
)
|
386
|
-
|
387
|
-
for k in highlighted_results.keys():
|
388
|
-
highlighted_results[k]["path"] = os.path.relpath(
|
389
|
-
highlighted_results[k]["path"], self.pipeline_reports
|
390
|
-
)
|
391
|
-
|
392
|
-
links = []
|
393
|
-
file_results = fetch_pipeline_results(
|
394
|
-
project=self.prj,
|
395
|
-
pipeline_name=self.pipeline_name,
|
396
|
-
sample_name=sample_name,
|
397
|
-
inclusion_fun=lambda x: x == "file",
|
398
|
-
)
|
399
|
-
for result_id, result in file_results.items():
|
400
|
-
desc = (
|
401
|
-
self.schema[result_id]["description"]
|
402
|
-
if "description" in self.schema[result_id]
|
403
|
-
else ""
|
404
|
-
)
|
405
|
-
links.append(
|
406
|
-
[
|
407
|
-
f"<b>{result['title']}</b>: {desc}",
|
408
|
-
os.path.relpath(result["path"], self.pipeline_reports),
|
409
|
-
]
|
410
|
-
)
|
411
|
-
image_results = fetch_pipeline_results(
|
412
|
-
project=self.prj,
|
413
|
-
pipeline_name=self.pipeline_name,
|
414
|
-
sample_name=sample_name,
|
415
|
-
inclusion_fun=lambda x: x == "image",
|
416
|
-
)
|
417
|
-
figures = []
|
418
|
-
for result_id, result in image_results.items():
|
419
|
-
figures.append(
|
420
|
-
[
|
421
|
-
os.path.relpath(result["path"], self.pipeline_reports),
|
422
|
-
result["title"],
|
423
|
-
os.patrh.relpath(result["thumbnail_path"], self.pipeline_reports),
|
424
|
-
]
|
425
|
-
)
|
426
|
-
|
427
|
-
template_vars = dict(
|
428
|
-
report_class="Sample",
|
429
|
-
navbar=navbar,
|
430
|
-
footer=footer,
|
431
|
-
sample_name=sample_name,
|
432
|
-
links=links,
|
433
|
-
figures=figures,
|
434
|
-
button_class=button_class,
|
435
|
-
sample_stats=sample_stats,
|
436
|
-
flag=flag,
|
437
|
-
highlighted_results=highlighted_results,
|
438
|
-
pipeline_name=self.pipeline_name,
|
439
|
-
amendments=self.prj.amendments,
|
440
|
-
)
|
441
|
-
_LOGGER.debug(f"sample.html | template_vars:\n{template_vars}")
|
442
|
-
save_html(
|
443
|
-
html_page, render_jinja_template("sample.html", self.j_env, template_vars)
|
444
|
-
)
|
445
|
-
return html_page
|
446
|
-
|
447
|
-
def create_status_html(self, status_table, navbar, footer):
|
448
|
-
"""
|
449
|
-
Generates a page listing all the samples, their run status, their
|
450
|
-
log file, and the total runtime if completed.
|
451
|
-
|
452
|
-
:param str navbar: HTML to be included as the navbar in the main summary page
|
453
|
-
:param str footer: HTML to be included as the footer
|
454
|
-
:return str: rendered status HTML file
|
455
|
-
"""
|
456
|
-
_LOGGER.debug("Building status page...")
|
457
|
-
template_vars = dict(status_table=status_table, navbar=navbar, footer=footer)
|
458
|
-
_LOGGER.debug(f"status.html | template_vars:\n{template_vars}")
|
459
|
-
return render_jinja_template("status.html", self.j_env, template_vars)
|
460
|
-
|
461
|
-
def create_index_html(self, navbar, footer):
|
462
|
-
"""
|
463
|
-
Generate an index.html style project home page w/ sample summary
|
464
|
-
statistics
|
465
|
-
|
466
|
-
:param str navbar: HTML to be included as the navbar in the main
|
467
|
-
summary page
|
468
|
-
:param str footer: HTML to be included as the footer
|
469
|
-
"""
|
470
|
-
# set default encoding when running in python2
|
471
|
-
if sys.version[0] == "2":
|
472
|
-
from importlib import reload
|
473
|
-
|
474
|
-
reload(sys)
|
475
|
-
sys.setdefaultencoding("utf-8")
|
476
|
-
_LOGGER.info(f"Building index page for pipeline: {self.pipeline_name}")
|
477
|
-
|
478
|
-
# Add stats_summary.tsv button link
|
479
|
-
stats_file_path = get_file_for_project(
|
480
|
-
self.prj, self.pipeline_name, "stats_summary.tsv"
|
481
|
-
)
|
482
|
-
stats_file_path = (
|
483
|
-
os.path.relpath(stats_file_path, self.pipeline_reports)
|
484
|
-
if os.path.exists(stats_file_path)
|
485
|
-
else None
|
486
|
-
)
|
487
|
-
|
488
|
-
# Add objects_summary.yaml button link
|
489
|
-
objs_file_path = get_file_for_project(
|
490
|
-
self.prj, self.pipeline_name, "objs_summary.yaml"
|
491
|
-
)
|
492
|
-
objs_file_path = (
|
493
|
-
os.path.relpath(objs_file_path, self.pipeline_reports)
|
494
|
-
if os.path.exists(objs_file_path)
|
495
|
-
else None
|
496
|
-
)
|
497
|
-
|
498
|
-
# Add stats summary table to index page and produce individual
|
499
|
-
# sample pages
|
500
|
-
# Produce table rows
|
501
|
-
table_row_data = []
|
502
|
-
_LOGGER.info(" * Creating sample pages")
|
503
|
-
for sample in self.prj.samples:
|
504
|
-
sample_stat_results = fetch_pipeline_results(
|
505
|
-
project=self.prj,
|
506
|
-
pipeline_name=self.pipeline_name,
|
507
|
-
sample_name=sample.sample_name,
|
508
|
-
inclusion_fun=lambda x: x not in OBJECT_TYPES,
|
509
|
-
casting_fun=str,
|
510
|
-
)
|
511
|
-
sample_html = self.create_sample_html(
|
512
|
-
sample_stat_results, navbar, footer, sample.sample_name
|
513
|
-
)
|
514
|
-
rel_sample_html = os.path.relpath(sample_html, self.pipeline_reports)
|
515
|
-
# treat sample_name column differently - will need to provide
|
516
|
-
# a link to the sample page
|
517
|
-
table_cell_data = [[rel_sample_html, sample.sample_name]]
|
518
|
-
table_cell_data += list(sample_stat_results.values())
|
519
|
-
table_row_data.append(table_cell_data)
|
520
|
-
# Create parent samples page with links to each sample
|
521
|
-
save_html(
|
522
|
-
path=os.path.join(self.pipeline_reports, "samples.html"),
|
523
|
-
template=self.create_sample_parent_html(navbar, footer),
|
524
|
-
)
|
525
|
-
_LOGGER.info(" * Creating object pages")
|
526
|
-
# Create objects pages
|
527
|
-
self.create_object_htmls(navbar, footer)
|
528
|
-
|
529
|
-
# Create parent objects page with links to each object type
|
530
|
-
save_html(
|
531
|
-
path=os.path.join(self.pipeline_reports, "objects.html"),
|
532
|
-
template=self.create_object_parent_html(navbar, footer),
|
533
|
-
)
|
534
|
-
# Create status page with each sample's status listed
|
535
|
-
status_tab = create_status_table(
|
536
|
-
pipeline_name=self.pipeline_name,
|
537
|
-
project=self.prj,
|
538
|
-
pipeline_reports_dir=self.pipeline_reports,
|
539
|
-
)
|
540
|
-
save_html(
|
541
|
-
path=os.path.join(self.pipeline_reports, "status.html"),
|
542
|
-
template=self.create_status_html(status_tab, navbar, footer),
|
543
|
-
)
|
544
|
-
# Complete and close HTML file
|
545
|
-
columns = [self.prj.sample_table_index] + list(sample_stat_results.keys())
|
546
|
-
template_vars = dict(
|
547
|
-
navbar=navbar,
|
548
|
-
stats_file_path=stats_file_path,
|
549
|
-
objs_file_path=objs_file_path,
|
550
|
-
columns=columns,
|
551
|
-
columns_json=dumps(columns),
|
552
|
-
table_row_data=table_row_data,
|
553
|
-
project_name=self.prj.name,
|
554
|
-
pipeline_name=self.pipeline_name,
|
555
|
-
stats_json=self._stats_to_json_str(),
|
556
|
-
footer=footer,
|
557
|
-
amendments=self.prj.amendments,
|
558
|
-
)
|
559
|
-
_LOGGER.debug(f"index.html | template_vars:\n{template_vars}")
|
560
|
-
save_html(
|
561
|
-
self.index_html_path,
|
562
|
-
render_jinja_template("index.html", self.j_env, template_vars),
|
563
|
-
)
|
564
|
-
|
565
|
-
def get_nonhighlighted_results(self, types):
|
566
|
-
"""
|
567
|
-
Get a list of non-highlighted results in the schema
|
568
|
-
|
569
|
-
:param list[str] types: types to narrow down the results
|
570
|
-
:return list[str]: result ID that are of the requested type and
|
571
|
-
are not highlighted
|
572
|
-
"""
|
573
|
-
results = []
|
574
|
-
for k, v in self.schema.items():
|
575
|
-
if self.schema[k]["type"] in types:
|
576
|
-
if "highlight" not in self.schema[k].keys():
|
577
|
-
results.append(k)
|
578
|
-
# intentionally "== False" to exclude "falsy" values
|
579
|
-
elif self.schema[k]["highlight"] == False:
|
580
|
-
results.append(k)
|
581
|
-
return results
|
582
|
-
|
583
|
-
def _stats_to_json_str(self):
|
584
|
-
results = {}
|
585
|
-
for sample in self.prj.samples:
|
586
|
-
results[sample.sample_name] = fetch_pipeline_results(
|
587
|
-
project=self.prj,
|
588
|
-
sample_name=sample.sample_name,
|
589
|
-
pipeline_name=self.pipeline_name,
|
590
|
-
inclusion_fun=lambda x: x not in OBJECT_TYPES,
|
591
|
-
casting_fun=str,
|
592
|
-
)
|
593
|
-
return dumps(results)
|
594
|
-
|
595
|
-
def _get_navbar_dropdown_data_objects(self, objs, wd, context):
|
596
|
-
if objs is None or len(objs) == 0:
|
597
|
-
return None, None
|
598
|
-
relpaths = []
|
599
|
-
displayable_ids = []
|
600
|
-
for obj_id in objs:
|
601
|
-
displayable_ids.append(obj_id.replace("_", " "))
|
602
|
-
page_name = os.path.join(
|
603
|
-
self.pipeline_reports, (obj_id + ".html").replace(" ", "_").lower()
|
604
|
-
)
|
605
|
-
relpaths.append(_make_relpath(page_name, wd, context))
|
606
|
-
return relpaths, displayable_ids
|
607
|
-
|
608
|
-
def _get_navbar_dropdown_data_samples(self, wd, context):
|
609
|
-
relpaths = []
|
610
|
-
sample_names = []
|
611
|
-
for sample in self.prj.samples:
|
612
|
-
page_name = os.path.join(
|
613
|
-
self.pipeline_reports,
|
614
|
-
f"{sample.sample_name}.html".replace(" ", "_").lower(),
|
615
|
-
)
|
616
|
-
relpaths.append(_make_relpath(page_name, wd, context))
|
617
|
-
sample_names.append(sample.sample_name)
|
618
|
-
return relpaths, sample_names
|
619
|
-
|
620
|
-
|
621
|
-
def render_jinja_template(name, jinja_env, args=dict()):
|
622
|
-
"""
|
623
|
-
Render template in the specified jinja environment using the provided args
|
624
|
-
|
625
|
-
:param str name: name of the template
|
626
|
-
:param dict args: arguments to pass to the template
|
627
|
-
:param jinja2.Environment jinja_env: the initialized environment to use in
|
628
|
-
this the looper HTML reports context
|
629
|
-
:return str: rendered template
|
630
|
-
"""
|
631
|
-
assert isinstance(args, dict), "args has to be a dict"
|
632
|
-
template = jinja_env.get_template(name)
|
633
|
-
return template.render(**args)
|
634
|
-
|
635
|
-
|
636
|
-
def save_html(path, template):
|
637
|
-
"""
|
638
|
-
Save rendered template as an HTML file
|
639
|
-
|
640
|
-
:param str path: the desired location for the file to be produced
|
641
|
-
:param str template: the template or just string
|
642
|
-
"""
|
643
|
-
if not os.path.exists(os.path.dirname(path)):
|
644
|
-
os.makedirs(os.path.dirname(path))
|
645
|
-
try:
|
646
|
-
with open(path, "w") as f:
|
647
|
-
f.write(template)
|
648
|
-
except IOError:
|
649
|
-
_LOGGER.error("Could not write the HTML file: {}".format(path))
|
650
|
-
|
651
|
-
|
652
|
-
def get_jinja_env(templates_dirname=None):
|
653
|
-
"""
|
654
|
-
Create jinja environment with the provided path to the templates directory
|
655
|
-
|
656
|
-
:param str templates_dirname: path to the templates directory
|
657
|
-
:return jinja2.Environment: jinja environment
|
658
|
-
"""
|
659
|
-
if templates_dirname is None:
|
660
|
-
file_dir = os.path.dirname(os.path.realpath(__file__))
|
661
|
-
templates_dirname = os.path.join(file_dir, TEMPLATES_DIRNAME)
|
662
|
-
_LOGGER.debug("Using templates dir: " + templates_dirname)
|
663
|
-
return jinja2.Environment(loader=jinja2.FileSystemLoader(templates_dirname))
|
664
|
-
|
665
|
-
|
666
|
-
def _get_file_for_sample(
|
667
|
-
prj, sample_name, appendix, pipeline_name=None, basename=False
|
668
|
-
):
|
669
|
-
"""
|
670
|
-
Safely looks for files matching the appendix in the specified
|
671
|
-
location for the sample
|
672
|
-
|
673
|
-
:param str sample_name: name of the sample that the file name
|
674
|
-
should be found for
|
675
|
-
:param str appendix: the ending pecific for the file
|
676
|
-
:param bool basename: whether to return basename only
|
677
|
-
:return str: the name of the matched file
|
678
|
-
"""
|
679
|
-
fp = os.path.join(prj.results_folder, sample_name)
|
680
|
-
prepend_name = ""
|
681
|
-
if pipeline_name:
|
682
|
-
prepend_name += pipeline_name
|
683
|
-
if hasattr(prj, AMENDMENTS_KEY) and getattr(prj, AMENDMENTS_KEY):
|
684
|
-
prepend_name += f"_{'_'.join(getattr(prj, AMENDMENTS_KEY))}"
|
685
|
-
prepend_name = prepend_name + "_" if prepend_name else ""
|
686
|
-
fp = os.path.join(fp, f"{prepend_name}{appendix}")
|
687
|
-
if os.path.exists(fp):
|
688
|
-
return os.path.basename(fp) if basename else fp
|
689
|
-
raise FileNotFoundError(fp)
|
690
|
-
|
691
|
-
|
692
|
-
def _get_relpath_to_file(file_name, sample_name, location, relative_to):
|
693
|
-
"""
|
694
|
-
Safely gets the relative path for the file for the specified sample
|
695
|
-
|
696
|
-
:param str file_name: name of the file
|
697
|
-
:param str sample_name: name of the sample that the file path
|
698
|
-
should be found for
|
699
|
-
:param str location: where to look for the file
|
700
|
-
:param str relative_to: path the result path should be relative to
|
701
|
-
:return str: a path to the file
|
702
|
-
"""
|
703
|
-
abs_file_path = os.path.join(location, sample_name, file_name)
|
704
|
-
rel_file_path = os.path.relpath(abs_file_path, relative_to)
|
705
|
-
if file_name is None or not os.path.exists(abs_file_path):
|
706
|
-
return None
|
707
|
-
return rel_file_path
|
708
|
-
|
709
|
-
|
710
|
-
def _make_relpath(file_name, wd, context=None):
|
711
|
-
"""
|
712
|
-
Create a path relative to the context. This function introduces the
|
713
|
-
flexibility to the navbar links creation, which the can be used outside
|
714
|
-
of the native looper summary pages.
|
715
|
-
|
716
|
-
:param str file_name: the path to make relative
|
717
|
-
:param str wd: the dir the path should be relative to
|
718
|
-
:param list[str] context: the context the links will be used in. The
|
719
|
-
sequence of directories to be prepended to the HTML
|
720
|
-
file in the resulting navbar
|
721
|
-
:return str: relative path
|
722
|
-
"""
|
723
|
-
relpath = os.path.relpath(file_name, wd)
|
724
|
-
return relpath if not context else os.path.join(os.path.join(*context), relpath)
|
725
|
-
|
726
|
-
|
727
|
-
def _read_csv_encodings(path, encodings=["utf-8", "ascii"], **kwargs):
|
728
|
-
"""
|
729
|
-
Try to read file with the provided encodings
|
730
|
-
|
731
|
-
:param str path: path to file
|
732
|
-
:param list encodings: list of encodings to try
|
733
|
-
"""
|
734
|
-
idx = 0
|
735
|
-
while idx < len(encodings):
|
736
|
-
e = encodings[idx]
|
737
|
-
try:
|
738
|
-
t = _pd.read_csv(path, encoding=e, **kwargs)
|
739
|
-
return t
|
740
|
-
except UnicodeDecodeError:
|
741
|
-
pass
|
742
|
-
idx = idx + 1
|
743
|
-
_LOGGER.warning(
|
744
|
-
f"Could not read the log file '{path}' with encodings '{encodings}'"
|
745
|
-
)
|
746
|
-
|
747
|
-
|
748
|
-
def _read_tsv_to_json(path):
|
749
|
-
"""
|
750
|
-
Read a tsv file to a JSON formatted string
|
751
|
-
|
752
|
-
:param path: to file path
|
753
|
-
:return str: JSON formatted string
|
754
|
-
"""
|
755
|
-
assert os.path.exists(path), "The file '{}' does not exist".format(path)
|
756
|
-
_LOGGER.debug("Reading TSV from '{}'".format(path))
|
757
|
-
df = _pd.read_csv(path, sep="\t", index_col=False, header=None)
|
758
|
-
return df.to_json()
|
759
|
-
|
760
|
-
|
761
|
-
def fetch_pipeline_results(
|
762
|
-
project,
|
763
|
-
pipeline_name,
|
764
|
-
sample_name=None,
|
765
|
-
inclusion_fun=None,
|
766
|
-
casting_fun=None,
|
767
|
-
highlighted=False,
|
768
|
-
):
|
769
|
-
"""
|
770
|
-
Get the specific pipeline results for sample based on inclusion function
|
771
|
-
|
772
|
-
:param looper.Project project: project to get the results for
|
773
|
-
:param str pipeline_name: pipeline ID
|
774
|
-
:param str sample_name: sample ID
|
775
|
-
:param callable(str) inclusion_fun: a function that determines whether the
|
776
|
-
result should be returned based on it's type. Example input that the
|
777
|
-
function will be fed with is: 'image' or 'integer'
|
778
|
-
:param callable(str) casting_fun: a function that will be used to cast the
|
779
|
-
each of the results to a proper type before returning, e.g int, str
|
780
|
-
:param bool highlighted: return the highlighted or regular results
|
781
|
-
:return dict: selected pipeline results
|
782
|
-
"""
|
783
|
-
psms = project.get_pipestat_managers(
|
784
|
-
sample_name=sample_name, project_level=sample_name is None
|
785
|
-
)
|
786
|
-
if pipeline_name not in psms:
|
787
|
-
_LOGGER.warning(
|
788
|
-
f"Pipeline name '{pipeline_name}' not found in "
|
789
|
-
f"{list(psms.keys())}. This pipeline was not run for"
|
790
|
-
f" sample: {sample_name}"
|
791
|
-
)
|
792
|
-
return
|
793
|
-
# set defaults to arg functions
|
794
|
-
pass_all_fun = lambda x: x
|
795
|
-
inclusion_fun = inclusion_fun or pass_all_fun
|
796
|
-
casting_fun = casting_fun or pass_all_fun
|
797
|
-
psm = psms[pipeline_name]
|
798
|
-
# exclude object-like results from the stats results mapping
|
799
|
-
# TODO: can't rely on .data property being there
|
800
|
-
rep_data = psm.retrieve()
|
801
|
-
# rep_data = psm.data[psm.namespace][psm.record_identifier].items()
|
802
|
-
results = {
|
803
|
-
k: casting_fun(v)
|
804
|
-
for k, v in rep_data.items()
|
805
|
-
if k in psm.schema and inclusion_fun(psm.schema[k]["type"])
|
806
|
-
}
|
807
|
-
if highlighted:
|
808
|
-
return {k: v for k, v in results.items() if k in psm.highlighted_results}
|
809
|
-
return {k: v for k, v in results.items() if k not in psm.highlighted_results}
|
810
|
-
|
811
|
-
|
812
|
-
def uniqify(seq):
|
813
|
-
"""Fast way to uniqify while preserving input order."""
|
814
|
-
# http://stackoverflow.com/questions/480214/
|
815
|
-
seen = set()
|
816
|
-
seen_add = seen.add
|
817
|
-
return [x for x in seq if not (x in seen or seen_add(x))]
|
818
|
-
|
819
|
-
|
820
|
-
def create_status_table(project, pipeline_name, pipeline_reports_dir):
|
821
|
-
"""
|
822
|
-
Creates status table, the core of the status page.
|
823
|
-
|
824
|
-
:return str: rendered status HTML file
|
825
|
-
"""
|
826
|
-
|
827
|
-
def _rgb2hex(r, g, b):
|
828
|
-
return "#{:02x}{:02x}{:02x}".format(r, g, b)
|
829
|
-
|
830
|
-
def _warn(what, e, sn):
|
831
|
-
_LOGGER.warning(
|
832
|
-
f"Caught exception: {e}\n"
|
833
|
-
f"Could not determine {what} for sample: {sn}. "
|
834
|
-
f"Not reported or pipestat status schema is faulty."
|
835
|
-
)
|
836
|
-
|
837
|
-
log_paths = []
|
838
|
-
log_link_names = []
|
839
|
-
sample_paths = []
|
840
|
-
sample_names = []
|
841
|
-
statuses = []
|
842
|
-
status_styles = []
|
843
|
-
times = []
|
844
|
-
mems = []
|
845
|
-
status_descs = []
|
846
|
-
for sample in project.samples:
|
847
|
-
psms = project.get_pipestat_managers(sample_name=sample.sample_name)
|
848
|
-
psm = psms[pipeline_name]
|
849
|
-
sample_names.append(sample.sample_name)
|
850
|
-
# status and status style
|
851
|
-
try:
|
852
|
-
status = psm.get_status()
|
853
|
-
statuses.append(status)
|
854
|
-
status_metadata = psm.status_schema[status]
|
855
|
-
status_styles.append(_rgb2hex(*status_metadata["color"]))
|
856
|
-
status_descs.append(status_metadata["description"])
|
857
|
-
except Exception as e:
|
858
|
-
_warn("status", e, sample.sample_name)
|
859
|
-
statuses.append(NO_DATA_PLACEHOLDER)
|
860
|
-
status_styles.append(NO_DATA_PLACEHOLDER)
|
861
|
-
status_descs.append(NO_DATA_PLACEHOLDER)
|
862
|
-
sample_paths.append(f"{sample.sample_name}.html".replace(" ", "_").lower())
|
863
|
-
# log file path
|
864
|
-
try:
|
865
|
-
log = psm.retrieve(result_identifier="log")["path"]
|
866
|
-
assert os.path.exists(log), FileNotFoundError(f"Not found: {log}")
|
867
|
-
log_link_names.append(os.path.basename(log))
|
868
|
-
log_paths.append(os.path.relpath(log, pipeline_reports_dir))
|
869
|
-
except Exception as e:
|
870
|
-
_warn("log", e, sample.sample_name)
|
871
|
-
log_link_names.append(NO_DATA_PLACEHOLDER)
|
872
|
-
log_paths.append("")
|
873
|
-
# runtime and peak mem
|
874
|
-
try:
|
875
|
-
profile = psm.retrieve(result_identifier="profile")["path"]
|
876
|
-
assert os.path.exists(profile), FileNotFoundError(f"Not found: {profile}")
|
877
|
-
df = _pd.read_csv(profile, sep="\t", comment="#", names=PROFILE_COLNAMES)
|
878
|
-
df["runtime"] = _pd.to_timedelta(df["runtime"])
|
879
|
-
times.append(_get_runtime(df))
|
880
|
-
mems.append(_get_maxmem(df))
|
881
|
-
except Exception as e:
|
882
|
-
_warn("profile", e, sample.sample_name)
|
883
|
-
times.append(NO_DATA_PLACEHOLDER)
|
884
|
-
mems.append(NO_DATA_PLACEHOLDER)
|
885
|
-
|
886
|
-
template_vars = dict(
|
887
|
-
sample_names=sample_names,
|
888
|
-
log_paths=log_paths,
|
889
|
-
status_styles=status_styles,
|
890
|
-
statuses=statuses,
|
891
|
-
times=times,
|
892
|
-
mems=mems,
|
893
|
-
sample_paths=sample_paths,
|
894
|
-
log_link_names=log_link_names,
|
895
|
-
status_descs=status_descs,
|
896
|
-
)
|
897
|
-
_LOGGER.debug(f"status_table.html | template_vars:\n{template_vars}")
|
898
|
-
return render_jinja_template("status_table.html", get_jinja_env(), template_vars)
|
899
|
-
|
900
|
-
|
901
|
-
def _get_maxmem(profile):
|
902
|
-
"""
|
903
|
-
Get current peak memory
|
904
|
-
|
905
|
-
:param pandas.core.frame.DataFrame profile: a data frame representing
|
906
|
-
the current profile.tsv for a sample
|
907
|
-
:return str: max memory
|
908
|
-
"""
|
909
|
-
return f"{str(max(profile['mem']) if not profile['mem'].empty else 0)} GB"
|
910
|
-
|
911
|
-
|
912
|
-
def _get_runtime(profile_df):
|
913
|
-
"""
|
914
|
-
Collect the unique and last duplicated runtimes, sum them and then
|
915
|
-
return in str format
|
916
|
-
|
917
|
-
:param pandas.core.frame.DataFrame profile_df: a data frame representing
|
918
|
-
the current profile.tsv for a sample
|
919
|
-
:return str: sum of runtimes
|
920
|
-
"""
|
921
|
-
unique_df = profile_df[~profile_df.duplicated("cid", keep="last").values]
|
922
|
-
return str(
|
923
|
-
timedelta(seconds=sum(unique_df["runtime"].apply(lambda x: x.total_seconds())))
|
924
|
-
).split(".")[0]
|