genhpf 1.0.2__tar.gz → 1.0.4__tar.gz

{genhpf-1.0.2/src/genhpf.egg-info → genhpf-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: genhpf
-Version: 1.0.2
+Version: 1.0.4
 Summary: GenHPF: General Healthcare Predictive Framework with Multi-task Multi-source Learning
 Author-email: Jungwoo Oh <ojw0123@kaist.ac.kr>, Kyunghoon Hur <pacesun@kaist.ac.kr>
 License: MIT license
@@ -18,6 +18,7 @@ Requires-Dist: h5pickle==0.4.2
 Requires-Dist: scikit-learn==1.6.1
 Requires-Dist: pandas==2.2.3
 Requires-Dist: polars==1.17.1
+Requires-Dist: pyarrow==17.0.0
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: black; extra == "dev"

{genhpf-1.0.2 → genhpf-1.0.4}/examples/test/genhpf/meds_hierarchical.yaml

@@ -10,12 +10,7 @@ meds:
   output_dir: ???
 
 checkpoint:
-  save_dir: checkpoints
-  best_checkpoint_metric: auroc
-  maximize_best_checkpoint_metric: true
-  save_interval: 1
-  keep_last_epochs: 5
-  patience: 10
+  load_checkpoint: ???
 
 dataset:
   data_format: meds
@@ -44,10 +39,6 @@ criterion:
   num_labels:
     - 1
 
-optimization:
-  max_epoch: 100
-  lr: 1e-4
-
 model:
   _name: genhpf_predictor
 

{genhpf-1.0.2 → genhpf-1.0.4}/pyproject.toml

@@ -29,6 +29,7 @@ dependencies = [
     "scikit-learn==1.6.1",
     "pandas==2.2.3",
     "polars==1.17.1",
+    "pyarrow==17.0.0",
 ]
 
 [tool.setuptools_scm]

{genhpf-1.0.2 → genhpf-1.0.4}/src/genhpf/scripts/preprocess/genhpf/main.py

@@ -151,7 +151,10 @@ def get_parser():
     return parser
 
 
-def main(args):
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
     if not os.path.exists(args.dest):
         os.makedirs(args.dest)
 
@@ -169,6 +172,4 @@ def main(args):
 
 
 if __name__ == "__main__":
-    parser = get_parser()
-    args = parser.parse_args()
-    main(args)
+    main()

{genhpf-1.0.2 → genhpf-1.0.4}/src/genhpf/scripts/preprocess/preprocess_meds.py

@@ -1,10 +1,10 @@
 import functools
+import logging
 import glob
 import multiprocessing
 import os
 import re
 import shutil
-import warnings
 from argparse import ArgumentParser
 from bisect import bisect_left, bisect_right
 from datetime import datetime
@@ -17,6 +17,9 @@ import polars as pl
 from tqdm import tqdm
 from transformers import AutoTokenizer
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 pool_manager = multiprocessing.Manager()
 warned_codes = pool_manager.list()
 
@@ -71,11 +74,24 @@ def get_parser():
         default="outputs",
         help="directory to save processed outputs.",
     )
+    parser.add_argument(
+        "--skip-if-exists",
+        action="store_true",
+        help="whether or not to skip the processing if the output directory already "
+        "exists.",
+    )
     parser.add_argument(
         "--rebase",
         action="store_true",
         help="whether or not to rebase the output directory if exists.",
     )
+    parser.add_argument(
+        "--debug",
+        type=bool,
+        default=False,
+        help="whether or not to enable the debug mode, which forces the script to be run with "
+        "only one worker."
+    )
     parser.add_argument(
         "--workers",
         metavar="N",
@@ -101,23 +117,26 @@ def get_parser():
     return parser
 
 
-def main(args):
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
     root_path = Path(args.root)
     output_dir = Path(args.output_dir)
     metadata_dir = Path(args.metadata_dir)
     mimic_dir = Path(args.mimic_dir) if args.mimic_dir is not None else None
 
-    if not output_dir.exists():
-        output_dir.mkdir()
+    num_workers = max(args.workers, 1)
+    if args.debug:
+        num_workers = 1
     else:
-        if args.rebase:
-            shutil.rmtree(output_dir)
-        if output_dir.exists():
-            raise ValueError(
-                f"File exists: '{str(output_dir.resolve())}'. If you want to rebase the "
-                "directory, please run the script with --rebase."
+        cpu_count = multiprocessing.cpu_count()
+        if num_workers > cpu_count:
+            logger.warning(
+                f"Number of workers (--workers) is greater than the number of available CPUs "
+                f"({cpu_count}). Setting the number of workers to {cpu_count}."
             )
-        output_dir.mkdir()
+            num_workers = cpu_count
 
     if root_path.is_dir():
         data_paths = glob.glob(str(root_path / "**/*.csv"), recursive=True)
@@ -128,6 +147,34 @@ def main(args):
     else:
         data_paths = [root_path]
 
+    if not output_dir.exists():
+        output_dir.mkdir()
+    else:
+        if args.rebase:
+            shutil.rmtree(output_dir)
+            output_dir.mkdir()
+        elif output_dir.exists():
+            if args.skip_if_exists:
+                ls = glob.glob(str(output_dir / "**/*"), recursive=True)
+                expected_files = []
+                for subset in set(os.path.dirname(x) for x in data_paths):
+                    expected_files.extend([
+                        os.path.join(str(output_dir), os.path.basename(subset), f"{i}.h5")
+                        for i in range(num_workers)
+                    ])
+                if set(expected_files).issubset(set(ls)):
+                    logger.info(
+                        f"Output directory already contains the expected files. Skipping the "
+                        "processing as --skip-if-exists is set. If you want to rebase the directory, "
+                        "please run the script with --rebase."
+                    )
+                    return
+            else:
+                raise ValueError(
+                    f"File exists: '{str(output_dir.resolve())}'. If you want to rebase the "
+                    "directory automatically, please run the script with --rebase."
+                )
+
     label_col_name = args.cohort_label_name
 
     tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")
@@ -295,7 +342,7 @@ def main(args):
                 codes_metadata,
                 output_dir,
                 output_name,
-                args.workers,
+                num_workers,
                 d_items,
                 d_labitems,
                 warned_codes,
@@ -303,27 +350,36 @@ def main(args):
             )
 
             # meds --> remed
-            print("Processing...")
-            if args.workers <= 1:
+            logger.info(f"Start processing {data_path}")
+            if num_workers <= 1:
                 length_per_subject_gathered = [meds_to_remed_partial(data)]
                 del data
             else:
                 subject_ids = data["subject_id"].unique().to_list()
-                n = args.workers
+                n = num_workers
                 subject_id_chunks = [subject_ids[i::n] for i in range(n)]
                 data_chunks = []
                 for subject_id_chunk in subject_id_chunks:
                     data_chunks.append(data.filter(pl.col("subject_id").is_in(subject_id_chunk)))
                 del data
-                pool = multiprocessing.get_context("spawn").Pool(processes=args.workers)
+
+                num_valid_data_chunks = sum(map(lambda x: len(x) > 0, data_chunks))
+                if num_valid_data_chunks < num_workers:
+                    raise ValueError(
+                        "Number of valid data chunks (= number of unique subjects) were smaller "
+                        "than the specified num workers (--workers) due to the small size of data. "
+                        "Consider reducing the number of workers."
+                    )
+
+                pool = multiprocessing.get_context("spawn").Pool(processes=num_workers)
                 # the order is preserved
                 length_per_subject_gathered = pool.map(meds_to_remed_partial, data_chunks)
                 pool.close()
                 pool.join()
                 del data_chunks
 
-            if len(length_per_subject_gathered) != args.workers:
-                print(
+            if len(length_per_subject_gathered) != num_workers:
+                raise ValueError(
                     "Number of processed workers were smaller than the specified num workers "
                     "(--workers) due to the small size of data. Consider reducing the number of "
                     "workers."
@@ -393,7 +449,7 @@ def meds_to_remed(
 
                             if do_break and col_event not in warned_codes:
                                 warned_codes.append(col_event)
-                                warnings.warn(
+                                logger.warning(
                                     "The dataset contains some codes that are not specified in "
                                     "the codes metadata, which may not be intended. Note that we "
                                     f"process this code as it is for now: {col_event}."
@@ -579,6 +635,4 @@ def meds_to_remed(
 
 
 if __name__ == "__main__":
-    parser = get_parser()
-    args = parser.parse_args()
-    main(args)
+    main()

{genhpf-1.0.2 → genhpf-1.0.4/src/genhpf.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: genhpf
-Version: 1.0.2
+Version: 1.0.4
 Summary: GenHPF: General Healthcare Predictive Framework with Multi-task Multi-source Learning
 Author-email: Jungwoo Oh <ojw0123@kaist.ac.kr>, Kyunghoon Hur <pacesun@kaist.ac.kr>
 License: MIT license
@@ -18,6 +18,7 @@ Requires-Dist: h5pickle==0.4.2
 Requires-Dist: scikit-learn==1.6.1
 Requires-Dist: pandas==2.2.3
 Requires-Dist: polars==1.17.1
+Requires-Dist: pyarrow==17.0.0
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: black; extra == "dev"

{genhpf-1.0.2 → genhpf-1.0.4}/src/genhpf.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@
 LICENSE
 README.md
 pyproject.toml
-requirements.txt
 examples/pretrain/mlm/genhpf/flattened_pt.yaml
 examples/pretrain/simclr/genhpf/genhpf_hierarchical_pt.yaml
 examples/pretrain/wav2vec2/genhpf/hierarchical_pt.yaml

{genhpf-1.0.2 → genhpf-1.0.4}/src/genhpf.egg-info/requires.txt

@@ -6,6 +6,7 @@ h5pickle==0.4.2
 scikit-learn==1.6.1
 pandas==2.2.3
 polars==1.17.1
+pyarrow==17.0.0
 
 [dev]
 pre-commit

genhpf-1.0.2/requirements.txt

@@ -1,48 +0,0 @@
-appdirs==1.4.4
-axial-positional-embedding==0.2.1
-cachetools==5.3.1
-certifi==2023.5.7
-charset-normalizer==3.1.0
-click==8.1.3
-docker-pycreds==0.4.0
-einops==0.6.1
-filelock==3.12.0
-fsspec==2023.5.0
-gitdb==4.0.10
-GitPython==3.1.31
-h5pickle==0.4.2
-h5py==3.8.0
-huggingface-hub==0.14.1
-idna==3.4
-joblib==1.2.0
-local-attention==1.8.6
-numpy==1.24.3
-packaging==23.1
-pandas==2.0.2
-pathtools==0.1.2
-performer-pytorch==1.1.4
-Pillow==9.5.0
-protobuf==4.23.2
-psutil==5.9.5
-python-dateutil==2.8.2
-pytz==2023.3
-PyYAML==6.0
-regex==2023.5.5
-requests==2.31.0
-scikit-learn==1.2.2
-scipy==1.10.1
-sentry-sdk==1.24.0
-setproctitle==1.3.2
-six==1.16.0
-smmap==5.0.0
-threadpoolctl==3.1.0
-tokenizers==0.13.3
-tqdm==4.65.0
-transformers==4.29.2
-typing_extensions==4.6.2
-tzdata==2023.3
-urllib3==1.26.16
-wandb==0.15.3
-pyspark==3.5.0
-pyarrow==14.0.1
-treelib==1.7.0