virtool-workflow 6.0.0a2__py3-none-any.whl → 6.0.0a4__py3-none-any.whl

virtool_workflow/data/indexes.py

@@ -5,6 +5,7 @@ from pathlib import Path
 
 import aiofiles
 from pyfixtures import fixture
+from structlog import get_logger
 from virtool_core.models.analysis import Analysis
 from virtool_core.models.index import Index
 from virtool_core.models.job import Job
@@ -15,6 +16,8 @@ from virtool_workflow.api.client import APIClient
 from virtool_workflow.files import VirtoolFileFormat
 from virtool_workflow.errors import MissingJobArgument
 
+logger = get_logger("api")
+
 
 @dataclass
 class WFIndex:
@@ -125,7 +128,12 @@ class WFIndex:
 
 class WFNewIndex:
     def __init__(
-        self, api: APIClient, index_id: str, manifest: dict[str, int | str], path: Path
+        self,
+        api: APIClient,
+        index_id: str,
+        manifest: dict[str, int | str],
+        path: Path,
+        reference: ReferenceNested,
     ):
         self._api = api
 
@@ -138,6 +146,9 @@ class WFNewIndex:
         self.path = path
         """The path to the index directory in the workflow's work directory."""
 
+        self.reference = reference
+        """The parent reference."""
+
     async def delete(self):
         await self._api.delete(f"/indexes/{self.id}")
 
@@ -174,7 +185,7 @@ class WFNewIndex:
         )
 
     @property
-    def json_path(self) -> Path:
+    def otus_json_path(self) -> Path:
         """
         The path to the JSON representation of the reference index in the workflow's
         work directory.
@@ -193,12 +204,20 @@ async def index(
     """The :class:`WFIndex` for the current analysis job."""
     id_ = analysis.index.id
 
+    log = logger.bind(id=id_, resource="index")
+
+    log.info("loading index")
+
     index_json = await _api.get_json(f"/indexes/{id_}")
     index_ = Index(**index_json)
 
+    log.info("got index json")
+
     index_work_path = work_path / "indexes" / index_.id
     await asyncio.to_thread(index_work_path.mkdir, parents=True, exist_ok=True)
 
+    log.info("created index directory")
+
     for name in (
         "otus.json.gz",
         "reference.json.gz",
@@ -211,6 +230,7 @@ async def index(
         "reference.rev.2.bt2",
     ):
         await _api.get_file(f"/indexes/{id_}/files/{name}", index_work_path / name)
+        log.info("downloaded index file", name=name)
 
     await asyncio.to_thread(
         decompress_file,
@@ -219,6 +239,8 @@ async def index(
         proc,
     )
 
+    log.info("decompressed reference fasta")
+
     json_path = index_work_path / "otus.json"
 
     await asyncio.to_thread(
@@ -228,6 +250,8 @@ async def index(
         proc,
     )
 
+    log.info("decompressed reference otus json")
+
     async with aiofiles.open(json_path) as f:
         data = json.loads(await f.read())
 
@@ -242,6 +266,8 @@ async def index(
                 sequence_otu_map[sequence_id] = otu["_id"]
                 sequence_lengths[sequence_id] = len(sequence["sequence"])
 
+    log.info("parsed and loaded maps from otus json")
+
     return WFIndex(
         id=id_,
         path=index_work_path,
@@ -253,7 +279,9 @@ async def index(
 
 
 @fixture
-async def new_index(_api: APIClient, job: Job, work_path: Path) -> WFNewIndex:
+async def new_index(
+    _api: APIClient, job: Job, proc: int, work_path: Path
+) -> WFNewIndex:
     """
     The :class:`.WFNewIndex` for an index being created by the current job.
     """
@@ -262,19 +290,36 @@ async def new_index(_api: APIClient, job: Job, work_path: Path) -> WFNewIndex:
     except KeyError:
         raise MissingJobArgument("Missing jobs args key 'index_id'")
 
+    log = logger.bind(resource="new_index", id=id_, job_id=job.id)
+    log.info("loading index")
+
     index_json = await _api.get_json(f"/indexes/{id_}")
     index_ = Index(**index_json)
 
+    log.info("got index json")
+
     index_work_path = work_path / "indexes" / index_.id
     await asyncio.to_thread(index_work_path.mkdir, parents=True, exist_ok=True)
 
-    await _api.get_file(
-        f"/indexes/{id_}/files/otus.json.gz", index_work_path / "otus.json.gz"
+    log.info("created index directory")
+
+    compressed_otus_json_path = index_work_path / "otus.json.gz"
+    await _api.get_file(f"/indexes/{id_}/files/otus.json.gz", compressed_otus_json_path)
+    log.info("downloaded otus json")
+
+    await asyncio.to_thread(
+        decompress_file,
+        compressed_otus_json_path,
+        index_work_path / "otus.json",
+        processes=proc,
     )
 
+    log.info("decompressed otus json")
+
     return WFNewIndex(
         api=_api,
         index_id=id_,
         manifest=index_.manifest,
         path=index_work_path,
+        reference=index_.reference,
     )

virtool_workflow/data/jobs.py

@@ -49,7 +49,7 @@ async def push_status(
     payload = {
         "error": error,
         "progress": progress,
-        "stage": _step.function.__name__ if _step is not None else None,
+        "stage": _step.function.__name__ if _step is not None else "",
         "state": _state.value,
         "step_description": _step.description if _step is not None else "",
         "step_name": step_name,

virtool_workflow/data/samples.py

@@ -133,16 +133,24 @@ async def new_sample(
     _api: APIClient, job: Job, uploads: WFUploads, work_path: Path
 ) -> WFNewSample:
     """The sample associated with the current job."""
+
     id_ = job.args["sample_id"]
 
+    log = logger.bind(resource="sample", id=id_)
+    log.info("loading sample for sample creation")
+
     base_url_path = f"/samples/{id_}"
 
     sample_dict = await _api.get_json(base_url_path)
     sample = Sample(**sample_dict)
 
+    log.info("got sample json")
+
     uploads_path = work_path / "uploads"
     await asyncio.to_thread(uploads_path.mkdir, exist_ok=True, parents=True)
 
+    log.info("created uploads directory")
+
     files = tuple(
         WFNewSampleUpload(
             id=f["id"],
@@ -155,6 +163,8 @@ async def new_sample(
 
     await asyncio.gather(*[uploads.download(f.id, f.path) for f in files])
 
+    log.info("downloaded sample files")
+
     async def finalize(quality: dict[str, Any]):
         await _api.patch_json(base_url_path, data={"quality": quality})
 

{virtool_workflow-6.0.0a2.dist-info → virtool_workflow-6.0.0a4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virtool-workflow
-Version: 6.0.0a2
+Version: 6.0.0a4
 Summary: A framework for developing bioinformatics workflows for Virtool.
 Home-page: https://github.com/virtool/virtool-workflow
 License: MIT

{virtool_workflow-6.0.0a2.dist-info → virtool_workflow-6.0.0a4.dist-info}/RECORD

@@ -12,10 +12,10 @@ virtool_workflow/cli.py,sha256=yEl1LziABKbjc5MCOoUGLy-iuqehOou37x9ox4LA92M,1441
 virtool_workflow/data/__init__.py,sha256=L2bQ5rMpnCMuLWDMwel4-wPchfQCTHH3EEwxDqwGOsY,610
 virtool_workflow/data/analyses.py,sha256=UdNrQWAvWHZwPLJinx_2_OK5eBeRlObVEx2QHcbFuRk,3150
 virtool_workflow/data/hmms.py,sha256=-ANZkM3C7p25zCEeS4WJFskBvHZWcpKjFDBZC2Z9HEA,3271
-virtool_workflow/data/indexes.py,sha256=xYBANmOT6Sr0GXWfw4WWNcxnITpnX3cKjj4zmu00jdg,8004
-virtool_workflow/data/jobs.py,sha256=LIKTX4jWm5cbNbdf9Foxmur0uf7_BeJY0aLT5Hr0Pp0,1669
+virtool_workflow/data/indexes.py,sha256=woF5iKb011nj5Sq37wasKb-xf8gH-xugJSusqcV6Avk,9072
+virtool_workflow/data/jobs.py,sha256=VALeDaaT0WWqDyGsiNxbeKIcdklYZljWhu0zBhUPZs0,1667
 virtool_workflow/data/ml.py,sha256=Yoc4ePH-J_4BKi-wjCKFD6vuI58g3Jw7SKk4ri9ymQ4,1891
-virtool_workflow/data/samples.py,sha256=tysGIGFssX4APlcQU52NIvjbtn6uX77ATLYS7Qhcgfg,4696
+virtool_workflow/data/samples.py,sha256=p58-qwHsaPoa3oGiDM5yibVibuvH5JPhfx8zNuTq9KE,4915
 virtool_workflow/data/subtractions.py,sha256=X-R8t5gIboHVE7kHyziWntuoyb1SbTjSV9UETB0Tt38,7282
 virtool_workflow/data/uploads.py,sha256=sIpuS2AtJ5yvZWitkzqb6hG1iLVe7K4q6UIsEDizFCE,899
 virtool_workflow/decorators.py,sha256=mh4_wBVLcAQYkn_aPonYEb2TP7NxORVdjBVgfehjHwQ,1285
@@ -35,8 +35,8 @@ virtool_workflow/runtime/run_subprocess.py,sha256=AviofEsOvp0buXRST-IxAW2txLwoTV
 virtool_workflow/runtime/sentry.py,sha256=6WNUYeQCoofP49cCmEy5bn_JRc3CSQmy7pnDW3XnAbc,748
 virtool_workflow/utils.py,sha256=wJHdTcuwC3wzYfJVM32MJo1Xdn-6r7V1bJRoLr9mhU0,520
 virtool_workflow/workflow.py,sha256=RxboCKLyCpfikPLIw7QFss77ERNlw8tJZhIXFB88DBs,2678
-virtool_workflow-6.0.0a2.dist-info/LICENSE,sha256=nkoVQw9W4aoQM9zgtNzHDmztap5TuXZ1L2-87vNr3w8,1097
-virtool_workflow-6.0.0a2.dist-info/METADATA,sha256=5ai445zZ9mx32gzOES_OnSBLjYVNZ3vpS8HcAPGjF0Y,4321
-virtool_workflow-6.0.0a2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-virtool_workflow-6.0.0a2.dist-info/entry_points.txt,sha256=d4MA8ZDTJOU0jKZ3ymtHZbfLVoRPgItEIN5U4uIqay8,62
-virtool_workflow-6.0.0a2.dist-info/RECORD,,
+virtool_workflow-6.0.0a4.dist-info/LICENSE,sha256=nkoVQw9W4aoQM9zgtNzHDmztap5TuXZ1L2-87vNr3w8,1097
+virtool_workflow-6.0.0a4.dist-info/METADATA,sha256=QTa4cL-Od0i4l9lw78gSIP0s5HoVMv1_y6XeMY4j2vo,4321
+virtool_workflow-6.0.0a4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+virtool_workflow-6.0.0a4.dist-info/entry_points.txt,sha256=d4MA8ZDTJOU0jKZ3ymtHZbfLVoRPgItEIN5U4uIqay8,62
+virtool_workflow-6.0.0a4.dist-info/RECORD,,