looper 1.5.1__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

looper/utils.py

@@ -19,7 +19,7 @@ from pephubclient.constants import RegistryPath
 from pydantic.error_wrappers import ValidationError
 
 from .const import *
-from .exceptions import MisconfigurationException
+from .exceptions import MisconfigurationException, RegistryPathException
 
 _LOGGER = getLogger(__name__)
 
@@ -72,7 +72,7 @@ def fetch_flag_files(prj=None, results_folder="", flags=FLAGS):
     return files_by_flag
 
 
-def fetch_sample_flags(prj, sample, pl_name):
+def fetch_sample_flags(prj, sample, pl_name, flag_dir=None):
     """
     Find any flag files present for a sample associated with a project
 
@@ -82,7 +82,7 @@ def fetch_sample_flags(prj, sample, pl_name):
     :return Iterable[str]: collection of flag file path(s) associated with the
         given sample for the given project
     """
-    sfolder = sample_folder(prj=prj, sample=sample)
+    sfolder = flag_dir or sample_folder(prj=prj, sample=sample)
     if not os.path.isdir(sfolder):
         _LOGGER.debug(
             "Results folder ({}) doesn't exist for sample {}".format(
@@ -98,6 +98,29 @@ def fetch_sample_flags(prj, sample, pl_name):
     ]
 
 
+def get_sample_status(sample, flags):
+    """
+    get a sample status
+
+    """
+
+    statuses = []
+
+    for f in flags:
+        basename = os.path.basename(f)
+        status = os.path.splitext(basename)[0].split("_")[-1]
+        if sample in basename:
+            statuses.append(status.upper())
+
+    if len(statuses) > 1:
+        _LOGGER.warning(f"Multiple status flags found for {sample}")
+
+    if statuses == []:
+        return None
+
+    return statuses[0]
+
+
 def grab_project_data(prj):
     """
     From the given Project, grab Sample-independent data.
@@ -335,12 +358,12 @@ def init_generic_pipeline():
     # Destination one level down from CWD in pipeline folder
     dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
 
-    # Determine Generic Pipeline Interface
+    # Create Generic Pipeline Interface
     generic_pipeline_dict = {
-        "pipeline_name": "count_lines",
+        "pipeline_name": "default_pipeline_name",
         "pipeline_type": "sample",
         "output_schema": "output_schema.yaml",
-        "var_templates": {"pipeline": "{looper.piface_dir}/count_lines.sh"},
+        "var_templates": {"pipeline": "{looper.piface_dir}/pipeline.sh"},
         "command_template": "{pipeline.var_templates.pipeline} {sample.file} "
         "--output-parent {looper.sample_output_folder}",
     }
@@ -349,58 +372,101 @@ def init_generic_pipeline():
     if not os.path.exists(dest_file):
         with open(dest_file, "w") as file:
             yaml.dump(generic_pipeline_dict, file)
-        print(f"Generic pipeline interface successfully created at: {dest_file}")
+        print(f"Pipeline interface successfully created at: {dest_file}")
     else:
         print(
-            f"Generic pipeline interface file already exists `{dest_file}`. Skipping creation.."
+            f"Pipeline interface file already exists `{dest_file}`. Skipping creation.."
         )
 
+    # Create Generic Output Schema
+    dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
+    generic_output_schema_dict = {
+        "pipeline_name": "default_pipeline_name",
+        "samples": {
+            "number_of_lines": {
+                "type": "integer",
+                "description": "Number of lines in the input file.",
+            }
+        },
+    }
+    # Write file
+    if not os.path.exists(dest_file):
+        with open(dest_file, "w") as file:
+            yaml.dump(generic_output_schema_dict, file)
+        print(f"Output schema successfully created at: {dest_file}")
+    else:
+        print(f"Output schema file already exists `{dest_file}`. Skipping creation..")
+
+    # Create Generic countlines.sh
+    dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
+    shell_code = """#!/bin/bash
+linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
+pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
+echo "Number of lines: $linecount"
+    """
+    if not os.path.exists(dest_file):
+        with open(dest_file, "w") as file:
+            file.write(shell_code)
+        print(f"count_lines.sh successfully created at: {dest_file}")
+    else:
+        print(f"count_lines.sh file already exists `{dest_file}`. Skipping creation..")
+
     return True
 
 
-def init_dotfile(
-    path: str,
-    cfg_path: str = None,
+def read_looper_dotfile():
+    """
+    Read looper config file
+    :return str: path to the config file read from the dotfile
+    :raise MisconfigurationException: if the dotfile does not consist of the
+        required key pointing to the PEP
+    """
+    dot_file_path = dotfile_path(must_exist=True)
+    return read_looper_config_file(looper_config_path=dot_file_path)
+
+
+def initiate_looper_config(
+    looper_config_path: str,
+    pep_path: str = None,
     output_dir: str = None,
     sample_pipeline_interfaces: Union[List[str], str] = None,
     project_pipeline_interfaces: Union[List[str], str] = None,
     force=False,
 ):
     """
-    Initialize looper dotfile
+    Initialize looper config file
 
-    :param str path: absolute path to the file to initialize
-    :param str cfg_path: path to the config file. Absolute or relative to 'path'
+    :param str looper_config_path: absolute path to the file to initialize
+    :param str pep_path: path to the PEP to be used in pipeline
     :param str output_dir: path to the output directory
     :param str|list sample_pipeline_interfaces: path or list of paths to sample pipeline interfaces
     :param str|list project_pipeline_interfaces: path or list of paths to project pipeline interfaces
     :param bool force: whether the existing file should be overwritten
     :return bool: whether the file was initialized
     """
-    if os.path.exists(path) and not force:
-        print("Can't initialize, file exists: {}".format(path))
+    if os.path.exists(looper_config_path) and not force:
+        print(f"Can't initialize, file exists: {looper_config_path}")
         return False
-    if cfg_path:
-        if is_registry_path(cfg_path):
+
+    if pep_path:
+        if is_registry_path(pep_path):
             pass
         else:
-            cfg_path = expandpath(cfg_path)
-            if not os.path.isabs(cfg_path):
-                cfg_path = os.path.join(os.path.dirname(path), cfg_path)
-            assert os.path.exists(cfg_path), OSError(
+            pep_path = expandpath(pep_path)
+            if not os.path.isabs(pep_path):
+                pep_path = os.path.join(os.path.dirname(looper_config_path), pep_path)
+            assert os.path.exists(pep_path), OSError(
                 "Provided config path is invalid. You must provide path "
-                "that is either absolute or relative to: {}".format(
-                    os.path.dirname(path)
-                )
+                f"that is either absolute or relative to: {os.path.dirname(looper_config_path)}"
             )
     else:
-        cfg_path = "example/pep/path"
+        pep_path = "example/pep/path"
 
     if not output_dir:
         output_dir = "."
 
     looper_config_dict = {
-        "pep_config": os.path.relpath(cfg_path, os.path.dirname(path)),
+        "pep_config": os.path.relpath(pep_path),
         "output_dir": output_dir,
         "pipeline_interfaces": {
             "sample": sample_pipeline_interfaces,
@@ -408,24 +474,12 @@ def init_dotfile(
         },
     }
 
-    with open(path, "w") as dotfile:
+    with open(looper_config_path, "w") as dotfile:
         yaml.dump(looper_config_dict, dotfile)
-    print("Initialized looper dotfile: {}".format(path))
+    print(f"Initialized looper config file: {looper_config_path}")
     return True
 
 
-def read_looper_dotfile():
-    """
-    Read looper config file
-
-    :return str: path to the config file read from the dotfile
-    :raise MisconfigurationException: if the dotfile does not consist of the
-        required key pointing to the PEP
-    """
-    dot_file_path = dotfile_path(must_exist=True)
-    return read_looper_config_file(looper_config_path=dot_file_path)
-
-
 def read_looper_config_file(looper_config_path: str) -> dict:
     """
     Read Looper config file which includes:
@@ -442,7 +496,10 @@ def read_looper_config_file(looper_config_path: str) -> dict:
         dp_data = yaml.safe_load(dotfile)
 
     if PEP_CONFIG_KEY in dp_data:
+        # Looper expects the config path to live at looper.config_file
+        # However, user may wish to access the pep at looper.pep_config
         return_dict[PEP_CONFIG_FILE_KEY] = dp_data[PEP_CONFIG_KEY]
+        return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
 
     # TODO: delete it in looper 2.0
     elif DOTFILE_CFG_PTH_KEY in dp_data:
@@ -460,6 +517,9 @@ def read_looper_config_file(looper_config_path: str) -> dict:
             f"{OUTDIR_KEY} is not defined in looper config file ({looper_config_path})"
         )
 
+    if PIPESTAT_KEY in dp_data:
+        return_dict[PIPESTAT_KEY] = dp_data[PIPESTAT_KEY]
+
     if PIPELINE_INTERFACES_KEY in dp_data:
         dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
         return_dict[SAMPLE_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get("sample")
@@ -473,6 +533,17 @@ def read_looper_config_file(looper_config_path: str) -> dict:
         )
         dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
 
+    config_dir_path = os.path.dirname(os.path.abspath(looper_config_path))
+
+    # Expand paths in case ENV variables are used
+    for k, v in return_dict.items():
+        if isinstance(v, str):
+            v = expandpath(v)
+            if not os.path.isabs(v) and not is_registry_path(v):
+                return_dict[k] = os.path.join(config_dir_path, v)
+            else:
+                return_dict[k] = v
+
     return return_dict
 
 
@@ -510,8 +581,13 @@ def is_registry_path(input_string: str) -> bool:
     :param str input_string: path to the PEP (or registry path)
     :return bool: True if input is a registry path
     """
-    if input_string.endswith(".yaml"):
-        return False
+    try:
+        if input_string.endswith(".yaml"):
+            return False
+    except AttributeError:
+        raise RegistryPathException(
+            msg=f"Malformed registry path. Unable to parse {input_string} as a registry path."
+        )
     try:
         registry_path = RegistryPath(**parse_registry_path(input_string))
     except (ValidationError, TypeError):

{looper-1.5.1.dist-info → looper-1.6.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: looper
-Version: 1.5.1
+Version: 1.6.0a1
 Summary: A pipeline submission engine that parses sample inputs and submits pipelines for each sample.
 Home-page: https://github.com/pepkit/looper
 Author: Nathan Sheffield, Vince Reuter, Michal Stolarczyk, Johanna Klughammer, Andre Rendeiro
@@ -17,17 +17,17 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: colorama >=0.3.9
 Requires-Dist: divvy >=0.5.0
-Requires-Dist: eido >=0.2.0
+Requires-Dist: eido >=0.2.1
 Requires-Dist: jinja2
 Requires-Dist: logmuse >=0.2.0
 Requires-Dist: pandas >=2.0.2
-Requires-Dist: pephubclient
-Requires-Dist: peppy >=0.35.4
-Requires-Dist: pipestat >=0.5.1
+Requires-Dist: pephubclient >=0.1.2
+Requires-Dist: peppy >=0.40.0.a4
+Requires-Dist: pipestat >=v0.6.0a9
 Requires-Dist: pyyaml >=3.12
 Requires-Dist: rich >=9.10.0
 Requires-Dist: ubiquerg >=0.5.2
-Requires-Dist: yacman >=0.9
+Requires-Dist: yacman >=0.9.2
 
 # <img src="docs/img/looper_logo.svg" alt="looper logo" height="70">
 

{looper-1.5.1.dist-info → looper-1.6.0a1.dist-info}/RECORD

@@ -1,19 +1,19 @@
-looper/__init__.py,sha256=XGDsUX7q8eKekQcTs7ji_5Heubx7kri-dArEPGDROSs,17694
-looper/__main__.py,sha256=bHyvKMsHY0CIubHX7kHxJE1E9bQamu27kqF_puTFTyQ,230
-looper/_version.py,sha256=8rmxh34XkVT7yu5zgYupipfDrKs2hgGCjOMEIK3E4oI,22
-looper/conductor.py,sha256=I7rYEH08B2cUdHbLwtCElSPP2tomWQZxVZhuAqkrU08,35680
-looper/const.py,sha256=Almb7Cy_zRhH1Pb02cj8mFVLy8V1xzJst3LfkEyjE6c,7005
-looper/divvy.py,sha256=HcQ7p_Wk78jHzWgjM9saZpO8sBteA4EFBeUJYBr5zVw,21185
-looper/exceptions.py,sha256=QshRRtBkJ088n09iXn1G8SHfSNSDaGcqt4eISqYX_qg,2880
-looper/html_reports.py,sha256=lRsRIbk18otAb1sVcIsS9RD2vJwrv32GyEDRYSp5L8s,43820
-looper/html_reports_pipestat.py,sha256=TEWCMdLliP7WzzanwopSoIhTEvThbDJfuCT8n5P5Ij4,35130
-looper/html_reports_project_pipestat.py,sha256=U4AdyGrnnDnbKGFk-oh3A9rCjyNj43ZwppiZtXX92S0,10224
-looper/looper.py,sha256=e87oiGaWjXTlCZoXolCGy5eV8MS0nKl6MhfzHQJEWz4,46350
+looper/__init__.py,sha256=f_z9YY4ibOk7eyWoaViH_VaCXMlPQeiftbnibSFj-3E,1333
+looper/__main__.py,sha256=8CX2ae8mUQNI_Z8pdBT4i5UFqROFX1awyFnuYCKuYXg,238
+looper/_version.py,sha256=UG64J_OfEJF5nBy02cxP-9wbnmfesoV9F5GvRWXpSfM,24
+looper/cli_divvy.py,sha256=J07x83sqC4jJeu3_yS6KOARPWmwKGAV7JvN33T5zDac,5907
+looper/cli_looper.py,sha256=0SYyPAPg8MEGOnp_vjn0Mxohe7Annr56hwDBX_DqM5U,25769
+looper/conductor.py,sha256=BmMATwtkHoPSsffYopCQ1WxK83O3UjOTFRrakgD6DzA,30231
+looper/const.py,sha256=bPj4lTuj2l6gwHROWqj16iHfJFo9ghZAz8THNREWW4U,8558
+looper/divvy.py,sha256=qa1ebbQTfNupAyDfhfEJ6mbZ_V3zk-D_E-Tck7miJ38,15688
+looper/exceptions.py,sha256=r6SKKt-m8CXQnXGDnuiwoA6zBJhIZflygBKjX4RCloI,3419
+looper/looper.py,sha256=f__lSwDgzTcFpv5kUIrR-XdC49_YPvdVZTBfBdkm2J8,30251
 looper/parser_types.py,sha256=d3FHt54f9jo9VZMr5SQkbghcAdABqiYZW2JBGO5EBnw,2327
 looper/pipeline_interface.py,sha256=y46tB1_73d1FX8N1w4-GGvRBJ7rqhenuUYVtUfIhK5s,14974
-looper/processed_project.py,sha256=wZGtjmbx3qdMHqI56xyrZc-lmMsWv3mlRkjYYIL7W3o,9867
-looper/project.py,sha256=FjSfqaYPzjGmIgPPi6x--ymXhXrzDOI4BvJgjvrkKqA,31760
-looper/utils.py,sha256=N-eF0j3RFuGWjr7PRfYuVEvlyv4Nm6zqcpmCiUK-TlU,24519
+looper/plugins.py,sha256=MaMdPmK9U_4FkNJE5kccohBbY1i2qj1NTEucubFOJek,5747
+looper/processed_project.py,sha256=jZxoMYafvr-OHFxylc5ivGty1VwXBZhl0kgoFkY-174,9837
+looper/project.py,sha256=Ldsa__wLNnDpKFfN0cOCWQnL-pTuVrPbbHJPHFt9fcU,36196
+looper/utils.py,sha256=i7srIXPEnQjtNaoP0ziRpdYfB7HNY5_3rW5LoKIM15I,27257
 looper/default_config/divvy_config.yaml,sha256=wK5kLDGBV2wwoyqg2rl3X8SXjds4x0mwBUjUzF1Ln7g,1705
 looper/default_config/divvy_templates/localhost_bulker_template.sub,sha256=yn5VB9Brt7Hck9LT17hD2o8Kn-76gYJQk_A-8C1Gr4k,164
 looper/default_config/divvy_templates/localhost_docker_template.sub,sha256=XRr7AlR7-TP1L3hyBMfka_RgWRL9vzOlS5Kd1xSNwT0,183
@@ -55,9 +55,9 @@ looper/schemas/divvy_config_schema.yaml,sha256=7GJfKLc3VX4RGjHnOE1zxwsHXhj_ur9za
 looper/schemas/pipeline_interface_schema_generic.yaml,sha256=D16Rkpj03H9WnvA_N18iNU-hH_HwOuyESJ8Hk5hZSXc,1518
 looper/schemas/pipeline_interface_schema_project.yaml,sha256=-ZWyA0lKXWik3obuLNVk3IsAZYfbLVbCDvJnD-Fcluo,1567
 looper/schemas/pipeline_interface_schema_sample.yaml,sha256=x0OwVnijJpvm50DscvvJujdK4UAI7d71pqVemQS-D-0,1564
-looper-1.5.1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
-looper-1.5.1.dist-info/METADATA,sha256=bJ8Sgx37O3lxhAUWqNgz3ag1YO4ycs-m9QXaT3HB_Jw,1723
-looper-1.5.1.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-looper-1.5.1.dist-info/entry_points.txt,sha256=AEL1eb0gPLYvAEUewM35Ng4scXGZIWJK4Mxdj3Hm8Fw,83
-looper-1.5.1.dist-info/top_level.txt,sha256=I0Yf7djsoQAMzwHBbDiQi9hGtq4Z41_Ma5CX8qXG8Y8,7
-looper-1.5.1.dist-info/RECORD,,
+looper-1.6.0a1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
+looper-1.6.0a1.dist-info/METADATA,sha256=apZ1c5_AxHNy1MxlLtOxGrXRZzG3tYlvWFzZQKMw3kU,1741
+looper-1.6.0a1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+looper-1.6.0a1.dist-info/entry_points.txt,sha256=AEL1eb0gPLYvAEUewM35Ng4scXGZIWJK4Mxdj3Hm8Fw,83
+looper-1.6.0a1.dist-info/top_level.txt,sha256=I0Yf7djsoQAMzwHBbDiQi9hGtq4Z41_Ma5CX8qXG8Y8,7
+looper-1.6.0a1.dist-info/RECORD,,

{looper-1.5.1.dist-info → looper-1.6.0a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.1)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any