looper 1.5.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,924 +0,0 @@
1
- """ Generate HTML reports """
2
-
3
- import logging
4
- import os
5
- import sys
6
- from datetime import timedelta
7
- from json import dumps
8
-
9
- import jinja2
10
- import pandas as _pd
11
- from eido import read_schema
12
- from peppy.const import *
13
-
14
- from ._version import __version__ as v
15
- from .const import *
16
- from .utils import get_file_for_project
17
-
18
- _LOGGER = logging.getLogger("looper")
19
-
20
-
21
- class HTMLReportBuilder(object):
22
- """Generate HTML summary report for project/samples"""
23
-
24
- def __init__(self, prj):
25
- """
26
- The Project defines the instance.
27
-
28
- :param looper.Project prj: Project with which to work/operate on
29
- """
30
- super(HTMLReportBuilder, self).__init__()
31
- self.prj = prj
32
- self.j_env = get_jinja_env()
33
- self.output_dir = self.prj.output_dir
34
- self.reports_dir = os.path.join(self.output_dir, "reports")
35
- _LOGGER.debug(f"Reports dir: {self.reports_dir}")
36
-
37
- def __call__(self, pipeline_name, project_index_html=None):
38
- """
39
- Generate HTML report.
40
-
41
- :param str pipeline_name: ID of the pipeline to generate the report for
42
- :return str: path to the index page of the generated HTML report
43
- """
44
- # Generate HTML report
45
- self.pipeline_name = pipeline_name
46
- self.amendments_str = (
47
- "_".join(self.prj.amendments) if self.prj.amendments else ""
48
- )
49
- self.pipeline_reports = os.path.join(
50
- self.reports_dir,
51
- f"{self.pipeline_name}_{self.amendments_str}"
52
- if self.prj.amendments
53
- else self.pipeline_name,
54
- )
55
- self.prj_index_html_path = project_index_html
56
- self.index_html_path = os.path.join(self.pipeline_reports, "index.html")
57
- pifaces = self.prj.pipeline_interfaces
58
- selected_pipeline_pifaces = [
59
- p for p in pifaces if p.pipeline_name == self.pipeline_name
60
- ]
61
- schema_path = self.prj.get_schemas(
62
- selected_pipeline_pifaces, OUTPUT_SCHEMA_KEY
63
- )[0]
64
- self.schema = read_schema(schema_path)[0]
65
- navbar = self.create_navbar(
66
- navbar_links=self.create_navbar_links(
67
- wd=self.pipeline_reports,
68
- project_index_html_relpath=os.path.relpath(
69
- self.prj_index_html_path, self.pipeline_reports
70
- )
71
- if self.prj_index_html_path
72
- else None,
73
- ),
74
- index_html_relpath=os.path.relpath(
75
- self.index_html_path, self.pipeline_reports
76
- ),
77
- )
78
- self.create_index_html(navbar, self.create_footer())
79
- return self.index_html_path
80
-
81
- def create_object_parent_html(self, navbar, footer):
82
- """
83
- Generates a page listing all the project objects with links
84
- to individual object pages
85
-
86
- :param str navbar: HTML to be included as the navbar in the main summary page
87
- :param str footer: HTML to be included as the footer
88
- :return str: Rendered parent objects HTML file
89
- """
90
- if not os.path.exists(self.pipeline_reports):
91
- os.makedirs(self.pipeline_reports)
92
- pages = list()
93
- labels = list()
94
- obj_result_ids = self.get_nonhighlighted_results(OBJECT_TYPES)
95
-
96
- for key in obj_result_ids:
97
- desc = (
98
- self.schema[key]["description"]
99
- if "description" in self.schema[key]
100
- else ""
101
- )
102
- labels.append(f"<b>{key.replace('_', ' ')}</b>: {desc}")
103
- page_path = os.path.join(self.pipeline_reports, f"{key}.html".lower())
104
- pages.append(os.path.relpath(page_path, self.pipeline_reports))
105
-
106
- template_vars = dict(
107
- navbar=navbar, footer=footer, labels=labels, pages=pages, header="Objects"
108
- )
109
- _LOGGER.debug(
110
- f"object navbar_list_parent.html | template_vars:" f"\n{template_vars}"
111
- )
112
- return render_jinja_template(
113
- "navbar_list_parent.html", self.j_env, template_vars
114
- )
115
-
116
- def create_sample_parent_html(self, navbar, footer):
117
- """
118
- Generates a page listing all the project samples with links
119
- to individual sample pages
120
- :param str navbar: HTML to be included as the navbar in the main summary page
121
- :param str footer: HTML to be included as the footer
122
- :return str: Rendered parent samples HTML file
123
- """
124
- if not os.path.exists(self.pipeline_reports):
125
- os.makedirs(self.pipeline_reports)
126
- pages = list()
127
- labels = list()
128
- for sample in self.prj.samples:
129
- sample_name = str(sample.sample_name)
130
- sample_dir = os.path.join(self.prj.results_folder, sample_name)
131
-
132
- # Confirm sample directory exists, then build page
133
- if os.path.exists(sample_dir):
134
- page_path = os.path.join(
135
- self.pipeline_reports,
136
- f"{sample_name}.html".replace(" ", "_").lower(),
137
- )
138
- page_relpath = os.path.relpath(page_path, self.pipeline_reports)
139
- pages.append(page_relpath)
140
- labels.append(sample_name)
141
-
142
- template_vars = dict(
143
- navbar=navbar, footer=footer, labels=labels, pages=pages, header="Samples"
144
- )
145
- _LOGGER.debug(
146
- f"sample navbar_list_parent.html | template_vars:" f"\n{template_vars}"
147
- )
148
- return render_jinja_template(
149
- "navbar_list_parent.html", self.j_env, template_vars
150
- )
151
-
152
- def create_navbar(self, navbar_links, index_html_relpath):
153
- """
154
- Creates the navbar using the provided links
155
-
156
- :param str navbar_links: HTML list of links to be inserted into a navbar
157
- :return str: navbar HTML
158
- """
159
- template_vars = dict(navbar_links=navbar_links, index_html=index_html_relpath)
160
- return render_jinja_template("navbar.html", self.j_env, template_vars)
161
-
162
- def create_footer(self):
163
- """
164
- Renders the footer from the templates directory
165
-
166
- :return str: footer HTML
167
- """
168
- return render_jinja_template("footer.html", self.j_env, dict(version=v))
169
-
170
- def create_navbar_links(
171
- self, wd=None, context=None, project_index_html_relpath=None
172
- ):
173
- """
174
- Return a string containing the navbar prebuilt html.
175
-
176
- Generates links to each page relative to the directory of interest
177
- (wd arg) or uses the provided context to create the paths (context arg)
178
-
179
- :param path wd: the working directory of the current HTML page being
180
- generated, enables navbar links relative to page
181
- :param list[str] context: the context the links will be used in.
182
- The sequence of directories to be prepended to the HTML file in
183
- the resulting navbar
184
- :return str: navbar links as HTML-formatted string
185
- """
186
- # determine paths
187
- if wd is None and context is None:
188
- raise ValueError(
189
- "Either 'wd' (path the links should be relative to) or "
190
- "'context' (the context for the links) has to be provided."
191
- )
192
- status_relpath = _make_relpath(
193
- file_name=os.path.join(self.pipeline_reports, "status.html"),
194
- wd=wd,
195
- context=context,
196
- )
197
- objects_relpath = _make_relpath(
198
- file_name=os.path.join(self.pipeline_reports, "objects.html"),
199
- wd=wd,
200
- context=context,
201
- )
202
- samples_relpath = _make_relpath(
203
- file_name=os.path.join(self.pipeline_reports, "samples.html"),
204
- wd=wd,
205
- context=context,
206
- )
207
- # determine the outputs IDs by type
208
- obj_result_ids = self.get_nonhighlighted_results(OBJECT_TYPES)
209
- dropdown_keys_objects = None
210
- dropdown_relpaths_objects = None
211
- sample_names = None
212
- if len(obj_result_ids) > 0:
213
- # If the number of objects is 20 or less, use a drop-down menu
214
- if len(obj_result_ids) <= 20:
215
- (
216
- dropdown_relpaths_objects,
217
- dropdown_keys_objects,
218
- ) = self._get_navbar_dropdown_data_objects(
219
- objs=obj_result_ids, wd=wd, context=context
220
- )
221
- else:
222
- dropdown_relpaths_objects = objects_relpath
223
- if len(self.prj.samples) <= 20:
224
- (
225
- dropdown_relpaths_samples,
226
- sample_names,
227
- ) = self._get_navbar_dropdown_data_samples(wd=wd, context=context)
228
- else:
229
- # Create a menu link to the samples parent page
230
- dropdown_relpaths_samples = samples_relpath
231
- template_vars = dict(
232
- status_html_page=status_relpath,
233
- status_page_name="Status",
234
- dropdown_keys_objects=dropdown_keys_objects,
235
- objects_page_name="Objects",
236
- samples_page_name="Samples",
237
- objects_html_page=dropdown_relpaths_objects,
238
- samples_html_page=dropdown_relpaths_samples,
239
- menu_name_objects="Objects",
240
- menu_name_samples="Samples",
241
- sample_names=sample_names,
242
- all_samples=samples_relpath,
243
- all_objects=objects_relpath,
244
- sample_reports_parent=None,
245
- project_report=project_index_html_relpath,
246
- )
247
- _LOGGER.debug(f"navbar_links.html | template_vars:\n{template_vars}")
248
- return render_jinja_template("navbar_links.html", self.j_env, template_vars)
249
-
250
- def create_object_htmls(self, navbar, footer):
251
- """
252
- Generates a page for an individual object type with all of its
253
- plots from each sample
254
-
255
- :param str navbar: HTML to be included as the navbar in the main summary page
256
- :param str footer: HTML to be included as the footer
257
- """
258
- file_results = self.get_nonhighlighted_results(["file"])
259
- image_results = self.get_nonhighlighted_results(["image"])
260
-
261
- if not os.path.exists(self.pipeline_reports):
262
- os.makedirs(self.pipeline_reports)
263
- for file_result in file_results:
264
- links = []
265
- html_page_path = os.path.join(
266
- self.pipeline_reports, f"{file_result}.html".lower()
267
- )
268
- for sample in self.prj.samples:
269
- sample_result = fetch_pipeline_results(
270
- project=self.prj,
271
- pipeline_name=self.pipeline_name,
272
- sample_name=sample.sample_name,
273
- )
274
- if file_result not in sample_result:
275
- break
276
- sample_result = sample_result[file_result]
277
- links.append(
278
- [
279
- sample.sample_name,
280
- os.path.relpath(sample_result["path"], self.pipeline_reports),
281
- ]
282
- )
283
- else:
284
- link_desc = (
285
- self.schema[file_result]["description"]
286
- if "description" in self.schema[file_result]
287
- else "No description in schema"
288
- )
289
- template_vars = dict(
290
- navbar=navbar,
291
- footer=footer,
292
- name=sample_result["title"],
293
- figures=[],
294
- links=links,
295
- desc=link_desc,
296
- )
297
- save_html(
298
- html_page_path,
299
- render_jinja_template(
300
- "object.html", self.j_env, args=template_vars
301
- ),
302
- )
303
-
304
- for image_result in image_results:
305
- html_page_path = os.path.join(
306
- self.pipeline_reports, f"{image_result}.html".lower()
307
- )
308
- figures = []
309
- for sample in self.prj.samples:
310
- sample_result = fetch_pipeline_results(
311
- project=self.prj,
312
- pipeline_name=self.pipeline_name,
313
- sample_name=sample.sample_name,
314
- )
315
- if image_result not in sample_result:
316
- break
317
- sample_result = sample_result[image_result]
318
- figures.append(
319
- [
320
- os.path.relpath(sample_result["path"], self.pipeline_reports),
321
- sample.sample_name,
322
- os.path.relpath(
323
- sample_result["thumbnail_path"], self.pipeline_reports
324
- ),
325
- ]
326
- )
327
- else:
328
- img_desc = (
329
- self.schema[image_result]["description"]
330
- if "description" in self.schema[image_result]
331
- else "No description in schema"
332
- )
333
- template_vars = dict(
334
- navbar=navbar,
335
- footer=footer,
336
- name=sample_result["title"],
337
- figures=figures,
338
- links=[],
339
- desc=img_desc,
340
- )
341
- _LOGGER.debug(f"object.html | template_vars:\n{template_vars}")
342
- save_html(
343
- html_page_path,
344
- render_jinja_template(
345
- "object.html", self.j_env, args=template_vars
346
- ),
347
- )
348
-
349
- def create_sample_html(self, sample_stats, navbar, footer, sample_name):
350
- """
351
- Produce an HTML page containing all of a sample's objects
352
- and the sample summary statistics
353
-
354
- :param str sample_name: the name of the current sample
355
- :param dict sample_stats: pipeline run statistics for the current sample
356
- :param str navbar: HTML to be included as the navbar in the main summary page
357
- :param str footer: HTML to be included as the footer
358
- :return str: path to the produced HTML page
359
- """
360
- if not os.path.exists(self.pipeline_reports):
361
- os.makedirs(self.pipeline_reports)
362
- html_page = os.path.join(self.pipeline_reports, f"{sample_name}.html".lower())
363
-
364
- psms = self.prj.get_pipestat_managers(sample_name=sample_name)
365
- psm = psms[self.pipeline_name]
366
- flag = psm.get_status()
367
- if not flag:
368
- button_class = "btn btn-secondary"
369
- flag = "Missing"
370
- else:
371
- try:
372
- flag_dict = BUTTON_APPEARANCE_BY_FLAG[flag]
373
- except KeyError:
374
- button_class = "btn btn-secondary"
375
- flag = "Unknown"
376
- else:
377
- button_class = flag_dict["button_class"]
378
- flag = flag_dict["flag"]
379
- highlighted_results = fetch_pipeline_results(
380
- project=self.prj,
381
- pipeline_name=self.pipeline_name,
382
- sample_name=sample_name,
383
- inclusion_fun=lambda x: x == "file",
384
- highlighted=True,
385
- )
386
-
387
- for k in highlighted_results.keys():
388
- highlighted_results[k]["path"] = os.path.relpath(
389
- highlighted_results[k]["path"], self.pipeline_reports
390
- )
391
-
392
- links = []
393
- file_results = fetch_pipeline_results(
394
- project=self.prj,
395
- pipeline_name=self.pipeline_name,
396
- sample_name=sample_name,
397
- inclusion_fun=lambda x: x == "file",
398
- )
399
- for result_id, result in file_results.items():
400
- desc = (
401
- self.schema[result_id]["description"]
402
- if "description" in self.schema[result_id]
403
- else ""
404
- )
405
- links.append(
406
- [
407
- f"<b>{result['title']}</b>: {desc}",
408
- os.path.relpath(result["path"], self.pipeline_reports),
409
- ]
410
- )
411
- image_results = fetch_pipeline_results(
412
- project=self.prj,
413
- pipeline_name=self.pipeline_name,
414
- sample_name=sample_name,
415
- inclusion_fun=lambda x: x == "image",
416
- )
417
- figures = []
418
- for result_id, result in image_results.items():
419
- figures.append(
420
- [
421
- os.path.relpath(result["path"], self.pipeline_reports),
422
- result["title"],
423
- os.patrh.relpath(result["thumbnail_path"], self.pipeline_reports),
424
- ]
425
- )
426
-
427
- template_vars = dict(
428
- report_class="Sample",
429
- navbar=navbar,
430
- footer=footer,
431
- sample_name=sample_name,
432
- links=links,
433
- figures=figures,
434
- button_class=button_class,
435
- sample_stats=sample_stats,
436
- flag=flag,
437
- highlighted_results=highlighted_results,
438
- pipeline_name=self.pipeline_name,
439
- amendments=self.prj.amendments,
440
- )
441
- _LOGGER.debug(f"sample.html | template_vars:\n{template_vars}")
442
- save_html(
443
- html_page, render_jinja_template("sample.html", self.j_env, template_vars)
444
- )
445
- return html_page
446
-
447
- def create_status_html(self, status_table, navbar, footer):
448
- """
449
- Generates a page listing all the samples, their run status, their
450
- log file, and the total runtime if completed.
451
-
452
- :param str navbar: HTML to be included as the navbar in the main summary page
453
- :param str footer: HTML to be included as the footer
454
- :return str: rendered status HTML file
455
- """
456
- _LOGGER.debug("Building status page...")
457
- template_vars = dict(status_table=status_table, navbar=navbar, footer=footer)
458
- _LOGGER.debug(f"status.html | template_vars:\n{template_vars}")
459
- return render_jinja_template("status.html", self.j_env, template_vars)
460
-
461
- def create_index_html(self, navbar, footer):
462
- """
463
- Generate an index.html style project home page w/ sample summary
464
- statistics
465
-
466
- :param str navbar: HTML to be included as the navbar in the main
467
- summary page
468
- :param str footer: HTML to be included as the footer
469
- """
470
- # set default encoding when running in python2
471
- if sys.version[0] == "2":
472
- from importlib import reload
473
-
474
- reload(sys)
475
- sys.setdefaultencoding("utf-8")
476
- _LOGGER.info(f"Building index page for pipeline: {self.pipeline_name}")
477
-
478
- # Add stats_summary.tsv button link
479
- stats_file_path = get_file_for_project(
480
- self.prj, self.pipeline_name, "stats_summary.tsv"
481
- )
482
- stats_file_path = (
483
- os.path.relpath(stats_file_path, self.pipeline_reports)
484
- if os.path.exists(stats_file_path)
485
- else None
486
- )
487
-
488
- # Add objects_summary.yaml button link
489
- objs_file_path = get_file_for_project(
490
- self.prj, self.pipeline_name, "objs_summary.yaml"
491
- )
492
- objs_file_path = (
493
- os.path.relpath(objs_file_path, self.pipeline_reports)
494
- if os.path.exists(objs_file_path)
495
- else None
496
- )
497
-
498
- # Add stats summary table to index page and produce individual
499
- # sample pages
500
- # Produce table rows
501
- table_row_data = []
502
- _LOGGER.info(" * Creating sample pages")
503
- for sample in self.prj.samples:
504
- sample_stat_results = fetch_pipeline_results(
505
- project=self.prj,
506
- pipeline_name=self.pipeline_name,
507
- sample_name=sample.sample_name,
508
- inclusion_fun=lambda x: x not in OBJECT_TYPES,
509
- casting_fun=str,
510
- )
511
- sample_html = self.create_sample_html(
512
- sample_stat_results, navbar, footer, sample.sample_name
513
- )
514
- rel_sample_html = os.path.relpath(sample_html, self.pipeline_reports)
515
- # treat sample_name column differently - will need to provide
516
- # a link to the sample page
517
- table_cell_data = [[rel_sample_html, sample.sample_name]]
518
- table_cell_data += list(sample_stat_results.values())
519
- table_row_data.append(table_cell_data)
520
- # Create parent samples page with links to each sample
521
- save_html(
522
- path=os.path.join(self.pipeline_reports, "samples.html"),
523
- template=self.create_sample_parent_html(navbar, footer),
524
- )
525
- _LOGGER.info(" * Creating object pages")
526
- # Create objects pages
527
- self.create_object_htmls(navbar, footer)
528
-
529
- # Create parent objects page with links to each object type
530
- save_html(
531
- path=os.path.join(self.pipeline_reports, "objects.html"),
532
- template=self.create_object_parent_html(navbar, footer),
533
- )
534
- # Create status page with each sample's status listed
535
- status_tab = create_status_table(
536
- pipeline_name=self.pipeline_name,
537
- project=self.prj,
538
- pipeline_reports_dir=self.pipeline_reports,
539
- )
540
- save_html(
541
- path=os.path.join(self.pipeline_reports, "status.html"),
542
- template=self.create_status_html(status_tab, navbar, footer),
543
- )
544
- # Complete and close HTML file
545
- columns = [self.prj.sample_table_index] + list(sample_stat_results.keys())
546
- template_vars = dict(
547
- navbar=navbar,
548
- stats_file_path=stats_file_path,
549
- objs_file_path=objs_file_path,
550
- columns=columns,
551
- columns_json=dumps(columns),
552
- table_row_data=table_row_data,
553
- project_name=self.prj.name,
554
- pipeline_name=self.pipeline_name,
555
- stats_json=self._stats_to_json_str(),
556
- footer=footer,
557
- amendments=self.prj.amendments,
558
- )
559
- _LOGGER.debug(f"index.html | template_vars:\n{template_vars}")
560
- save_html(
561
- self.index_html_path,
562
- render_jinja_template("index.html", self.j_env, template_vars),
563
- )
564
-
565
- def get_nonhighlighted_results(self, types):
566
- """
567
- Get a list of non-highlighted results in the schema
568
-
569
- :param list[str] types: types to narrow down the results
570
- :return list[str]: result ID that are of the requested type and
571
- are not highlighted
572
- """
573
- results = []
574
- for k, v in self.schema.items():
575
- if self.schema[k]["type"] in types:
576
- if "highlight" not in self.schema[k].keys():
577
- results.append(k)
578
- # intentionally "== False" to exclude "falsy" values
579
- elif self.schema[k]["highlight"] == False:
580
- results.append(k)
581
- return results
582
-
583
- def _stats_to_json_str(self):
584
- results = {}
585
- for sample in self.prj.samples:
586
- results[sample.sample_name] = fetch_pipeline_results(
587
- project=self.prj,
588
- sample_name=sample.sample_name,
589
- pipeline_name=self.pipeline_name,
590
- inclusion_fun=lambda x: x not in OBJECT_TYPES,
591
- casting_fun=str,
592
- )
593
- return dumps(results)
594
-
595
- def _get_navbar_dropdown_data_objects(self, objs, wd, context):
596
- if objs is None or len(objs) == 0:
597
- return None, None
598
- relpaths = []
599
- displayable_ids = []
600
- for obj_id in objs:
601
- displayable_ids.append(obj_id.replace("_", " "))
602
- page_name = os.path.join(
603
- self.pipeline_reports, (obj_id + ".html").replace(" ", "_").lower()
604
- )
605
- relpaths.append(_make_relpath(page_name, wd, context))
606
- return relpaths, displayable_ids
607
-
608
- def _get_navbar_dropdown_data_samples(self, wd, context):
609
- relpaths = []
610
- sample_names = []
611
- for sample in self.prj.samples:
612
- page_name = os.path.join(
613
- self.pipeline_reports,
614
- f"{sample.sample_name}.html".replace(" ", "_").lower(),
615
- )
616
- relpaths.append(_make_relpath(page_name, wd, context))
617
- sample_names.append(sample.sample_name)
618
- return relpaths, sample_names
619
-
620
-
621
- def render_jinja_template(name, jinja_env, args=dict()):
622
- """
623
- Render template in the specified jinja environment using the provided args
624
-
625
- :param str name: name of the template
626
- :param dict args: arguments to pass to the template
627
- :param jinja2.Environment jinja_env: the initialized environment to use in
628
- this the looper HTML reports context
629
- :return str: rendered template
630
- """
631
- assert isinstance(args, dict), "args has to be a dict"
632
- template = jinja_env.get_template(name)
633
- return template.render(**args)
634
-
635
-
636
- def save_html(path, template):
637
- """
638
- Save rendered template as an HTML file
639
-
640
- :param str path: the desired location for the file to be produced
641
- :param str template: the template or just string
642
- """
643
- if not os.path.exists(os.path.dirname(path)):
644
- os.makedirs(os.path.dirname(path))
645
- try:
646
- with open(path, "w") as f:
647
- f.write(template)
648
- except IOError:
649
- _LOGGER.error("Could not write the HTML file: {}".format(path))
650
-
651
-
652
- def get_jinja_env(templates_dirname=None):
653
- """
654
- Create jinja environment with the provided path to the templates directory
655
-
656
- :param str templates_dirname: path to the templates directory
657
- :return jinja2.Environment: jinja environment
658
- """
659
- if templates_dirname is None:
660
- file_dir = os.path.dirname(os.path.realpath(__file__))
661
- templates_dirname = os.path.join(file_dir, TEMPLATES_DIRNAME)
662
- _LOGGER.debug("Using templates dir: " + templates_dirname)
663
- return jinja2.Environment(loader=jinja2.FileSystemLoader(templates_dirname))
664
-
665
-
666
- def _get_file_for_sample(
667
- prj, sample_name, appendix, pipeline_name=None, basename=False
668
- ):
669
- """
670
- Safely looks for files matching the appendix in the specified
671
- location for the sample
672
-
673
- :param str sample_name: name of the sample that the file name
674
- should be found for
675
- :param str appendix: the ending pecific for the file
676
- :param bool basename: whether to return basename only
677
- :return str: the name of the matched file
678
- """
679
- fp = os.path.join(prj.results_folder, sample_name)
680
- prepend_name = ""
681
- if pipeline_name:
682
- prepend_name += pipeline_name
683
- if hasattr(prj, AMENDMENTS_KEY) and getattr(prj, AMENDMENTS_KEY):
684
- prepend_name += f"_{'_'.join(getattr(prj, AMENDMENTS_KEY))}"
685
- prepend_name = prepend_name + "_" if prepend_name else ""
686
- fp = os.path.join(fp, f"{prepend_name}{appendix}")
687
- if os.path.exists(fp):
688
- return os.path.basename(fp) if basename else fp
689
- raise FileNotFoundError(fp)
690
-
691
-
692
- def _get_relpath_to_file(file_name, sample_name, location, relative_to):
693
- """
694
- Safely gets the relative path for the file for the specified sample
695
-
696
- :param str file_name: name of the file
697
- :param str sample_name: name of the sample that the file path
698
- should be found for
699
- :param str location: where to look for the file
700
- :param str relative_to: path the result path should be relative to
701
- :return str: a path to the file
702
- """
703
- abs_file_path = os.path.join(location, sample_name, file_name)
704
- rel_file_path = os.path.relpath(abs_file_path, relative_to)
705
- if file_name is None or not os.path.exists(abs_file_path):
706
- return None
707
- return rel_file_path
708
-
709
-
710
- def _make_relpath(file_name, wd, context=None):
711
- """
712
- Create a path relative to the context. This function introduces the
713
- flexibility to the navbar links creation, which the can be used outside
714
- of the native looper summary pages.
715
-
716
- :param str file_name: the path to make relative
717
- :param str wd: the dir the path should be relative to
718
- :param list[str] context: the context the links will be used in. The
719
- sequence of directories to be prepended to the HTML
720
- file in the resulting navbar
721
- :return str: relative path
722
- """
723
- relpath = os.path.relpath(file_name, wd)
724
- return relpath if not context else os.path.join(os.path.join(*context), relpath)
725
-
726
-
727
- def _read_csv_encodings(path, encodings=["utf-8", "ascii"], **kwargs):
728
- """
729
- Try to read file with the provided encodings
730
-
731
- :param str path: path to file
732
- :param list encodings: list of encodings to try
733
- """
734
- idx = 0
735
- while idx < len(encodings):
736
- e = encodings[idx]
737
- try:
738
- t = _pd.read_csv(path, encoding=e, **kwargs)
739
- return t
740
- except UnicodeDecodeError:
741
- pass
742
- idx = idx + 1
743
- _LOGGER.warning(
744
- f"Could not read the log file '{path}' with encodings '{encodings}'"
745
- )
746
-
747
-
748
- def _read_tsv_to_json(path):
749
- """
750
- Read a tsv file to a JSON formatted string
751
-
752
- :param path: to file path
753
- :return str: JSON formatted string
754
- """
755
- assert os.path.exists(path), "The file '{}' does not exist".format(path)
756
- _LOGGER.debug("Reading TSV from '{}'".format(path))
757
- df = _pd.read_csv(path, sep="\t", index_col=False, header=None)
758
- return df.to_json()
759
-
760
-
761
- def fetch_pipeline_results(
762
- project,
763
- pipeline_name,
764
- sample_name=None,
765
- inclusion_fun=None,
766
- casting_fun=None,
767
- highlighted=False,
768
- ):
769
- """
770
- Get the specific pipeline results for sample based on inclusion function
771
-
772
- :param looper.Project project: project to get the results for
773
- :param str pipeline_name: pipeline ID
774
- :param str sample_name: sample ID
775
- :param callable(str) inclusion_fun: a function that determines whether the
776
- result should be returned based on it's type. Example input that the
777
- function will be fed with is: 'image' or 'integer'
778
- :param callable(str) casting_fun: a function that will be used to cast the
779
- each of the results to a proper type before returning, e.g int, str
780
- :param bool highlighted: return the highlighted or regular results
781
- :return dict: selected pipeline results
782
- """
783
- psms = project.get_pipestat_managers(
784
- sample_name=sample_name, project_level=sample_name is None
785
- )
786
- if pipeline_name not in psms:
787
- _LOGGER.warning(
788
- f"Pipeline name '{pipeline_name}' not found in "
789
- f"{list(psms.keys())}. This pipeline was not run for"
790
- f" sample: {sample_name}"
791
- )
792
- return
793
- # set defaults to arg functions
794
- pass_all_fun = lambda x: x
795
- inclusion_fun = inclusion_fun or pass_all_fun
796
- casting_fun = casting_fun or pass_all_fun
797
- psm = psms[pipeline_name]
798
- # exclude object-like results from the stats results mapping
799
- # TODO: can't rely on .data property being there
800
- rep_data = psm.retrieve()
801
- # rep_data = psm.data[psm.namespace][psm.record_identifier].items()
802
- results = {
803
- k: casting_fun(v)
804
- for k, v in rep_data.items()
805
- if k in psm.schema and inclusion_fun(psm.schema[k]["type"])
806
- }
807
- if highlighted:
808
- return {k: v for k, v in results.items() if k in psm.highlighted_results}
809
- return {k: v for k, v in results.items() if k not in psm.highlighted_results}
810
-
811
-
812
- def uniqify(seq):
813
- """Fast way to uniqify while preserving input order."""
814
- # http://stackoverflow.com/questions/480214/
815
- seen = set()
816
- seen_add = seen.add
817
- return [x for x in seq if not (x in seen or seen_add(x))]
818
-
819
-
820
- def create_status_table(project, pipeline_name, pipeline_reports_dir):
821
- """
822
- Creates status table, the core of the status page.
823
-
824
- :return str: rendered status HTML file
825
- """
826
-
827
- def _rgb2hex(r, g, b):
828
- return "#{:02x}{:02x}{:02x}".format(r, g, b)
829
-
830
- def _warn(what, e, sn):
831
- _LOGGER.warning(
832
- f"Caught exception: {e}\n"
833
- f"Could not determine {what} for sample: {sn}. "
834
- f"Not reported or pipestat status schema is faulty."
835
- )
836
-
837
- log_paths = []
838
- log_link_names = []
839
- sample_paths = []
840
- sample_names = []
841
- statuses = []
842
- status_styles = []
843
- times = []
844
- mems = []
845
- status_descs = []
846
- for sample in project.samples:
847
- psms = project.get_pipestat_managers(sample_name=sample.sample_name)
848
- psm = psms[pipeline_name]
849
- sample_names.append(sample.sample_name)
850
- # status and status style
851
- try:
852
- status = psm.get_status()
853
- statuses.append(status)
854
- status_metadata = psm.status_schema[status]
855
- status_styles.append(_rgb2hex(*status_metadata["color"]))
856
- status_descs.append(status_metadata["description"])
857
- except Exception as e:
858
- _warn("status", e, sample.sample_name)
859
- statuses.append(NO_DATA_PLACEHOLDER)
860
- status_styles.append(NO_DATA_PLACEHOLDER)
861
- status_descs.append(NO_DATA_PLACEHOLDER)
862
- sample_paths.append(f"{sample.sample_name}.html".replace(" ", "_").lower())
863
- # log file path
864
- try:
865
- log = psm.retrieve(result_identifier="log")["path"]
866
- assert os.path.exists(log), FileNotFoundError(f"Not found: {log}")
867
- log_link_names.append(os.path.basename(log))
868
- log_paths.append(os.path.relpath(log, pipeline_reports_dir))
869
- except Exception as e:
870
- _warn("log", e, sample.sample_name)
871
- log_link_names.append(NO_DATA_PLACEHOLDER)
872
- log_paths.append("")
873
- # runtime and peak mem
874
- try:
875
- profile = psm.retrieve(result_identifier="profile")["path"]
876
- assert os.path.exists(profile), FileNotFoundError(f"Not found: {profile}")
877
- df = _pd.read_csv(profile, sep="\t", comment="#", names=PROFILE_COLNAMES)
878
- df["runtime"] = _pd.to_timedelta(df["runtime"])
879
- times.append(_get_runtime(df))
880
- mems.append(_get_maxmem(df))
881
- except Exception as e:
882
- _warn("profile", e, sample.sample_name)
883
- times.append(NO_DATA_PLACEHOLDER)
884
- mems.append(NO_DATA_PLACEHOLDER)
885
-
886
- template_vars = dict(
887
- sample_names=sample_names,
888
- log_paths=log_paths,
889
- status_styles=status_styles,
890
- statuses=statuses,
891
- times=times,
892
- mems=mems,
893
- sample_paths=sample_paths,
894
- log_link_names=log_link_names,
895
- status_descs=status_descs,
896
- )
897
- _LOGGER.debug(f"status_table.html | template_vars:\n{template_vars}")
898
- return render_jinja_template("status_table.html", get_jinja_env(), template_vars)
899
-
900
-
901
- def _get_maxmem(profile):
902
- """
903
- Get current peak memory
904
-
905
- :param pandas.core.frame.DataFrame profile: a data frame representing
906
- the current profile.tsv for a sample
907
- :return str: max memory
908
- """
909
- return f"{str(max(profile['mem']) if not profile['mem'].empty else 0)} GB"
910
-
911
-
912
- def _get_runtime(profile_df):
913
- """
914
- Collect the unique and last duplicated runtimes, sum them and then
915
- return in str format
916
-
917
- :param pandas.core.frame.DataFrame profile_df: a data frame representing
918
- the current profile.tsv for a sample
919
- :return str: sum of runtimes
920
- """
921
- unique_df = profile_df[~profile_df.duplicated("cid", keep="last").values]
922
- return str(
923
- timedelta(seconds=sum(unique_df["runtime"].apply(lambda x: x.total_seconds())))
924
- ).split(".")[0]