genhpf 1.0.11__py3-none-any.whl

genhpf/__init__.py

@@ -0,0 +1,9 @@
+"""isort:skip_file"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+__package_name__ = "genhpf"
+try:
+    __version__ = version(__package_name__)
+except PackageNotFoundError:
+    __version__ = "unknown"

genhpf/configs/__init__.py

@@ -0,0 +1,23 @@
+import logging
+
+from .configs import (
+    BaseConfig,
+    Config,
+    CommonConfig,
+    DistributedTrainingConfig,
+    DatasetConfig,
+    CheckpointConfig,
+)
+from .constants import ChoiceEnum
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "BaseConfig",
+    "Config",
+    "CommonConfig",
+    "DistributedTrainingConfig",
+    "DatasetConfig",
+    "CheckpointConfig",
+    "ChoiceEnum",
+]

genhpf/configs/config.yaml

@@ -0,0 +1,8 @@
+# @package _group_
+
+hydra:
+  run:
+    dir: .
+
+defaults:
+  - ...

genhpf/configs/configs.py

@@ -0,0 +1,240 @@
+from dataclasses import _MISSING_TYPE, dataclass, field
+from typing import Any, List, Optional
+
+from omegaconf import MISSING
+
+from genhpf.configs.constants import LOG_FORMAT_CHOICES
+
+
+@dataclass
+class BaseConfig:
+    """base configuration class"""
+
+    _name: Optional[str] = None
+
+    @staticmethod
+    def name():
+        return None
+
+    def _get_all_attributes(self) -> List[str]:
+        return [k for k in self.__dataclass_fields__.keys()]
+
+    def _get_meta(self, attribute_name: str, meta: str, default: Optional[Any] = None) -> Any:
+        return self.__dataclass_fields__[attribute_name].metadata.get(meta, default)
+
+    def _get_name(self, attribute_name: str) -> str:
+        return self.__dataclass_fields__[attribute_name].name
+
+    def _get_default(self, attribute_name: str) -> Any:
+        if hasattr(self, attribute_name):
+            if str(getattr(self, attribute_name)).startswith("${"):
+                return str(getattr(self, attribute_name))
+            elif str(self.__dataclass_fields__[attribute_name].default).startswith("${"):
+                return str(self.__dataclass_fields__[attribute_name].default)
+            elif getattr(self, attribute_name) != self.__dataclass_fields__[attribute_name].default:
+                return getattr(self, attribute_name)
+
+        f = self.__dataclass_fields__[attribute_name]
+        if not isinstance(f.default_factory, _MISSING_TYPE):
+            return f.default_factory()
+        return f.default
+
+    def _get_type(self, attribute_name: str) -> Any:
+        return self.__dataclass_fields__[attribute_name].type
+
+    def _get_help(self, attribute_name: str) -> Any:
+        return self._get_meta(attribute_name, "help")
+
+    def _get_argparse_const(self, attribute_name: str) -> Any:
+        return self._get_meta(attribute_name, "argparse_const")
+
+    def _get_argparse_alias(self, attribute_name: str) -> Any:
+        return self._get_meta(attribute_name, "argparse_alias")
+
+    def _get_choices(self, attribute_name: str) -> Any:
+        return self._get_meta(attribute_name, "choices")
+
+
+@dataclass
+class CommonConfig(BaseConfig):
+    debug: bool = field(default=False, metadata={"help": "enable debug mode"})
+    no_progress_bar: bool = field(default=False, metadata={"help": "disable progress bar"})
+    log_interval: int = field(default=100, metadata={"help": "log progress every N batches"})
+    log_format: Optional[LOG_FORMAT_CHOICES] = field(default=None, metadata={"help": "log format to use"})
+    log_file: Optional[str] = field(default=None, metadata={"help": "log file to copy metrics to."})
+    wandb_project: Optional[str] = field(
+        default=None, metadata={"help": "Weights and Biases project name to use for logging"}
+    )
+    wandb_entity: Optional[str] = field(
+        default=None, metadata={"help": "Weights and Biases entity(team) name to use for logging"}
+    )
+    seed: int = field(default=42, metadata={"help": "random seed"})
+    all_gather_list_size: int = field(
+        default=32768,
+        metadata={"help": "number of bytes reserved for gathering stats from workers"},
+    )
+
+
+@dataclass
+class DistributedTrainingConfig(BaseConfig):
+    distributed_world_size: int = field(default=1, metadata={"help": "total number of GPUs across all nodes"})
+    distributed_rank: Optional[int] = field(default=0, metadata={"help": "rank of the current worker"})
+    distributed_backend: str = field(default="nccl", metadata={"help": "distributed backend"})
+    distributed_init_method: Optional[str] = field(
+        default=None,
+        metadata={"help": "typically tcp://hostname:port that will be used to " "init distributed training"},
+    )
+    distributed_port: int = field(default=12355, metadata={"help": "port number for distributed training"})
+    device_id: int = field(default=0, metadata={"help": "which GPU to use"})
+    bucket_cap_mb: int = field(default=25, metadata={"help": "bucket size for reduction"})
+    find_unused_parameters: bool = field(
+        default=False, metadata={"help": "disable unused parameter detection when using distributed training"}
+    )
+    broadcast_buffers: bool = field(
+        default=False,
+        metadata={
+            "help": "Copy non-trainable parameters between GPUs, such as " "batchnorm population statistics"
+        },
+    )
+
+
+@dataclass
+class DatasetConfig(BaseConfig):
+    data_format: str = field(
+        default="genhpf", metadata={"help": "data format. supported formats: genhpf, meds"}
+    )
+    data: str = field(default=MISSING, metadata={"help": "path to the data directory"})
+    label: bool = field(default=False, metadata={"help": "whether to load labels from the dataset"})
+    vocab_size: int = field(default=MISSING, metadata={"help": "size of the vocabulary"})
+    pad_token_id: int = field(default=0, metadata={"help": "pad token id"})
+    sep_token_id: int = field(default=102, metadata={"help": "sep token id"})
+    dummy_token_id: int = field(default=101, metadata={"help": "dummy token id"})
+    ignore_index: int = field(
+        default=-100,
+        metadata={
+            "help": "specifies a target value that is ignored and does not contribute to "
+            "the input gradient. only applied to cross-entropy loss"
+        },
+    )
+    apply_mask: bool = field(default=False, metadata={"help": "whether to apply masking to the input tokens"})
+    mask_prob: float = field(default=0.15, metadata={"help": "probability for masking tokens"})
+    mask_unit: str = field(
+        default="token", metadata={"help": "unit for masking. supported units: token, event"}
+    )
+    mask_token_id: int = field(default=103, metadata={"help": "mask token id"})
+    num_workers: int = field(default=1, metadata={"help": "how many subprocesses to use for data loading"})
+    batch_size: Optional[int] = field(default=None, metadata={"help": "number of examples in a batch"})
+    train_subset: str = field(
+        default="train", metadata={"help": "data subset name to use for training (e.g., train, valid, test)"}
+    )
+    valid_subset: Optional[str] = field(
+        default="valid", metadata={"help": "comma separated list of data subset names to use for validation"}
+    )
+    test_subset: Optional[str] = field(
+        default="test", metadata={"help": "comma separated list of data subset names to use for test"}
+    )
+    combine_train_subsets: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": "whether to combine all training subsets into one dataset",
+            "argparse_alias": "--combine-train",
+        },
+    )
+    combine_valid_subsets: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": "whether to combine all validation subsets into one dataset",
+            "argparse_alias": "--combine-val",
+        },
+    )
+    combine_test_subsets: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": "whether to combine all test subsets into one dataset",
+            "argparse_alias": "--combine-test",
+        },
+    )
+    disable_validation: Optional[bool] = field(
+        default=None, metadata={"help": "whether to disable validation during training"}
+    )
+
+
+@dataclass
+class OptimizationConfig(BaseConfig):
+    max_epoch: int = field(default=0, metadata={"help": "maximum number of epochs to train"})
+    lr: float = field(default=1e-4, metadata={"help": "learning rate"})
+    adam_betas: Any = field(default=(0.9, 0.999), metadata={"help": "betas for Adam optimizer"})
+    adam_eps: float = field(default=1e-8, metadata={"help": "epsilon for Adam optimizer"})
+    weight_decay: float = field(default=1e-8, metadata={"help": "weight decay for Adam optimizer"})
+
+
+@dataclass
+class CheckpointConfig(BaseConfig):
+    save_dir: str = field(default="checkpoints", metadata={"help": "path to save checkpoints"})
+    checkpoint_prefix: str = field(
+        default="checkpoint", metadata={"help": "prefix to add to the checkpoint file name"}
+    )
+    checkpoint_suffix: str = field(default="", metadata={"help": "suffix to add to the checkpoint file name"})
+    load_checkpoint: Optional[str] = field(
+        default=None, metadata={"help": "path to a checkpoint to load model weights from, if provided"}
+    )
+    no_save: bool = field(default=False, metadata={"help": "don't save checkpoints"})
+    save_interval: int = field(default=1, metadata={"help": "save a checkpoint every N epochs"})
+    keep_last_epochs: int = field(default=-1, metadata={"help": "keep last N epoch checkpoints"})
+    no_last_checkpoints: bool = field(default=False, metadata={"help": "don't store last checkpoints"})
+    best_checkpoint_metric: str = field(
+        default="loss", metadata={"help": 'metric to use for saving "best" checkpoints'}
+    )
+    maximize_best_checkpoint_metric: bool = field(
+        default=False,
+        metadata={"help": 'select the largest metric value for saving "best" checkpoints'},
+    )
+    patience: int = field(
+        default=-1,
+        metadata={
+            "help": (
+                "early stop training if valid performance doesn't "
+                "improve for N consecutive validation runs; note "
+                "that this is influenced by --validate-interval"
+            )
+        },
+    )
+
+
+@dataclass
+class MEDSConfig(BaseConfig):
+    output_predictions: bool = field(
+        default=False,
+        metadata={
+            "help": "whether to output predictions. if turned on, `genhpf-test` will automatically "
+            "output predictions of the test set specified by `dataset.test_subset` in the "
+            "`meds.output_dir` directory."
+        },
+    )
+    labels_dir: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "a path to the label directory for MEDS dataset. this is required to store "
+            "output predictions in the format expected by `meds-evaluation` when "
+            "`meds.output_predictions` is turned on."
+        },
+    )
+    output_dir: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "a path to the output directory to store output predictions. "
+            "this is only used when `meds.output_predictions` is turned on."
+        },
+    )
+
+
+@dataclass
+class Config(BaseConfig):
+    common: CommonConfig = field(default_factory=CommonConfig)
+    distributed_training: DistributedTrainingConfig = field(default_factory=DistributedTrainingConfig)
+    dataset: DatasetConfig = field(default_factory=DatasetConfig)
+    optimization: OptimizationConfig = field(default_factory=OptimizationConfig)
+    checkpoint: CheckpointConfig = field(default_factory=CheckpointConfig)
+    meds: MEDSConfig = field(default_factory=MEDSConfig)
+    model: Any = MISSING
+    criterion: Any = None

genhpf/configs/constants.py

@@ -0,0 +1,29 @@
+from enum import Enum, EnumMeta
+from typing import List
+
+class StrEnumMeta(EnumMeta):
+    # this is workaround for submitit pickling leading to instance checks failing in hydra for StrEnum, see
+    # https://github.com/facebookresearch/hydra/issues/1156
+    @classmethod
+    def __instancecheck__(cls, other):
+        return "enum" in str(type(other))
+
+
+class StrEnum(Enum, metaclass=StrEnumMeta):
+    def __str__(self):
+        return self.value
+
+    def __eq__(self, other: str):
+        return self.value == other
+
+    def __repr__(self):
+        return self.value
+
+    def __hash__(self):
+        return hash(str(self))
+
+def ChoiceEnum(choices: List[str]):
+    """return the Enum class used to enforce list of choices"""
+    return StrEnum("Choices", {k: k for k in choices})
+
+LOG_FORMAT_CHOICES = ChoiceEnum(["json", "none", "simple", "tqdm", "csv"])

genhpf/configs/initialize.py

@@ -0,0 +1,58 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+from dataclasses import _MISSING_TYPE
+from hydra.core.config_store import ConfigStore
+from omegaconf import DictConfig, OmegaConf
+
+from genhpf.configs import Config
+
+logger = logging.getLogger(__name__)
+
+def hydra_init(cfg_name="config") -> None:
+    cs = ConfigStore.instance()
+    cs.store(name=cfg_name, node=Config)
+
+    for k in Config.__dataclass_fields__:
+        v = Config.__dataclass_fields__[k].default
+        if isinstance(v, _MISSING_TYPE):
+            v = Config.__dataclass_fields__[k].default_factory
+            if not isinstance(v, _MISSING_TYPE):
+                v = v()
+        try:
+            cs.store(name = k, node = v)
+        except BaseException:
+            logger.error(f"{k} - {v}")
+            raise
+
+def add_defaults(cfg: DictConfig) -> None:
+    """This function adds default values that are stored in dataclasses that hydra doesn't know about """
+
+    from genhpf.criterions import CRITERION_DATACLASS_REGISTRY
+    from genhpf.models import MODEL_DATACLASS_REGISTRY
+    from genhpf.configs.utils import merge_with_parent
+    from typing import Any
+
+    OmegaConf.set_struct(cfg, False)
+
+    for k, v in Config.__dataclass_fields__.items():
+        field_cfg = cfg.get(k)
+        if field_cfg is not None and v.type == Any:
+            dc = None
+
+            if isinstance(field_cfg, str):
+                field_cfg = DictConfig({"_name": field_cfg})
+                field_cfg.__dict__["_parent"] = field_cfg.__dict__["_parent"]
+
+            name = field_cfg.get("_name")
+
+            if k == "model":
+                dc = MODEL_DATACLASS_REGISTRY.get(name)
+            elif k == "criterion":
+                dc = CRITERION_DATACLASS_REGISTRY.get(name)
+            
+            if dc is not None:
+                cfg[k] = merge_with_parent(dc, field_cfg)

genhpf/configs/utils.py

@@ -0,0 +1,29 @@
+from dataclasses import is_dataclass
+from omegaconf import OmegaConf, open_dict
+
+from genhpf.configs import BaseConfig
+
+def merge_with_parent(dc: BaseConfig, cfg: BaseConfig, remove_missing=False):
+    if remove_missing:
+
+        def remove_missing_rec(src_keys, target_cfg):
+            if is_dataclass(target_cfg):
+                target_keys = set(target_cfg.__dataclass_fields__.keys())
+            else:
+                target_keys = set(target_cfg.keys())
+            
+            for k in list(src_keys.keys()):
+                if k not in target_keys:
+                    del src_keys[k]
+                elif OmegaConf.is_config(src_keys[k]):
+                    tgt = getattr(target_cfg, k)
+                    if tgt is not None and (is_dataclass(tgt) or hasattr(tgt, "keys")):
+                        remove_missing_rec(src_keys[k], tgt)
+    
+        with open_dict(cfg):
+            remove_missing_rec(cfg, dc)
+
+    merged_cfg = OmegaConf.merge(dc, cfg)
+    merged_cfg.__dict__["_parent"] = cfg.__dict__["_parent"]
+    OmegaConf.set_struct(merged_cfg, True)
+    return merged_cfg

genhpf/criterions/__init__.py

@@ -0,0 +1,74 @@
+import importlib
+import os
+
+from hydra.core.config_store import ConfigStore
+
+from genhpf.criterions.criterion import BaseCriterion #noqa
+from genhpf.configs.utils import merge_with_parent
+
+CRITERION_REGISTRY = {}
+CRITERION_DATACLASS_REGISTRY = {}
+
+def build_criterion(cfg) -> BaseCriterion:
+    criterion = None
+    criterion_type = getattr(cfg, "_name", None)
+
+    if criterion_type in CRITERION_REGISTRY:
+        criterion = CRITERION_REGISTRY[criterion_type]
+        # set defaults from dataclass
+        dc = CRITERION_DATACLASS_REGISTRY[criterion_type]
+        cfg = merge_with_parent(dc(), cfg)
+
+    assert criterion is not None, (
+        f"Could not infer criterion type from {str(criterion_type)}. "
+        + "Available criterions: "
+        + str(CRITERION_REGISTRY.keys())
+        + " Requested criterion type: "
+        + str(criterion_type)
+    )
+
+    return criterion.build_criterion(cfg)
+
+def register_criterion(name, dataclass=None):
+    """
+    New criterion types can be added with the :func:`register_criterion`
+    function decorator.
+
+    Args:
+        name (str): the name of the criterion
+    """
+
+    def register_criterion_cls(cls):
+        if name in CRITERION_REGISTRY:
+            raise ValueError(f"Cannot register duplicate criterion ({name})")
+        if not issubclass(cls, BaseCriterion):
+            raise ValueError(
+                f"Criterion ({name}: {cls.__name__}) must extend Base Criterion"
+            )
+        CRITERION_REGISTRY[name] = cls
+        if dataclass is not None:
+            CRITERION_DATACLASS_REGISTRY[name] = dataclass
+
+            cs = ConfigStore.instance()
+            node = dataclass()
+            node._name = name
+            cs.store(name=name, group="criterion", node=node, provider="genhpf")
+
+        return cls
+
+    return register_criterion_cls
+
+def import_criterions(criterions_dir, namespace):
+    for file in os.listdir(criterions_dir):
+        path = os.path.join(criterions_dir, file)
+        if (
+            not file.startswith("_")
+            and not file.startswith(".")
+            and (file.endswith(".py") or os.path.isdir(path))
+        ):
+            criterion_name = file[: file.find(".py")] if file.endswith(".py") else file
+            importlib.import_module(namespace + "." + criterion_name)
+
+# automatically import any Python files in the criterions/ directory
+criterions_dir = os.path.dirname(__file__)
+import_criterions(criterions_dir, "genhpf.criterions")

genhpf/criterions/binary_cross_entropy.py

@@ -0,0 +1,114 @@
+import math
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Tuple
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+import genhpf.utils.utils as utils
+from genhpf.criterions import BaseCriterion, register_criterion
+from genhpf.criterions.criterion import CriterionConfig
+from genhpf.loggings import meters, metrics
+from genhpf.loggings.meters import safe_round
+
+
+@dataclass
+class BinaryCrossEntropyConfig(CriterionConfig):
+    threshold: float = field(default=0.5, metadata={"help": "threshold value for binary classification"})
+
+
+@register_criterion("binary_cross_entropy", dataclass=BinaryCrossEntropyConfig)
+class BinaryCrossEntropy(BaseCriterion):
+    def __init__(self, cfg: BinaryCrossEntropyConfig):
+        super().__init__(cfg)
+
+        if self.task_names is not None and len(self.task_names) > 1:
+            raise ValueError(
+                "binary_cross_entropy only supports single task training. if you want "
+                " to train multiple tasks, use multi_task_criterion instead."
+            )
+
+        self.threshold = cfg.threshold
+
+    def compute_loss(
+        self, logits: torch.Tensor, targets: torch.Tensor, sample=None, net_output=None, model=None
+    ) -> Tuple[torch.Tensor, List[float]]:
+        logits = logits.flatten()
+        targets = targets.float()
+        probs = torch.sigmoid(logits)
+        loss = F.binary_cross_entropy(input=probs, target=targets, reduction="sum")
+        return loss, [loss.detach().item()]
+
+    def get_sample_size(self, sample, targets: torch.Tensor) -> int:
+        if "sample_size" in sample:
+            sample_size = sample["sample_size"]
+        else:
+            sample_size = targets.numel()
+        return sample_size
+
+    def get_logging_outputs(
+        self, logging_output, logits: torch.Tensor, targets: torch.Tensor, sample=None
+    ) -> List[Dict[str, Any]]:
+        with torch.no_grad():
+            probs = torch.sigmoid(logits.flatten())
+            outputs = probs > self.threshold
+
+            if probs.numel() == 0:
+                corr = 0
+                count = 0
+            else:
+                count = float(probs.numel())
+                corr = (outputs == targets).sum().item()
+
+            logging_output["correct"] = corr
+            logging_output["count"] = count
+
+            # report aucs only in eval mode
+            if not self.training:
+                logging_output["_y_true"] = targets.cpu().numpy()
+                logging_output["_y_score"] = probs.cpu().numpy()
+
+        return logging_output
+
+    @staticmethod
+    def reduce_metrics(logging_outputs: List[Dict[str, Any]], prefix: str = None) -> None:
+        """Aggregate logging outputs from data parallel training."""
+        if prefix is None:
+            prefix = ""
+        elif prefix is not None and not prefix.endswith("_"):
+            prefix = prefix + "_"
+
+        loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
+
+        sample_size = utils.item(sum(log.get("sample_size", 0) for log in logging_outputs))
+
+        metrics.log_scalar(f"{prefix}loss", loss_sum / (sample_size or 1) / math.log(2), sample_size, round=3)
+
+        if "_y_true" in logging_outputs[0] and "_y_score" in logging_outputs[0]:
+            y_true = np.concatenate([log.get("_y_true", []) for log in logging_outputs])
+            y_score = np.concatenate([log.get("_y_score", []) for log in logging_outputs])
+
+            metrics.log_custom(meters.AUCMeter, f"_{prefix}auc", y_score, y_true)
+
+        correct = sum(log.get("correct", 0) for log in logging_outputs)
+        metrics.log_scalar(f"_{prefix}correct", correct)
+
+        total = sum(log.get("count", 0) for log in logging_outputs)
+        metrics.log_scalar(f"_{prefix}total", total)
+
+        if total > 0:
+            metrics.log_derived(
+                f"{prefix}accuracy",
+                lambda meters: safe_round(meters[f"_{prefix}correct"].sum / meters[f"_{prefix}total"].sum, 5)
+                if meters[f"_{prefix}total"].sum > 0
+                else float("nan"),
+            )
+
+    def post_validate(self, stats, agg, **kwargs):
+        for key in agg.keys():
+            if key.startswith("_") and key.endswith("auc"):
+                stats[key[1:-3] + "auroc"] = agg[key].auroc
+                stats[key[1:-3] + "auprc"] = agg[key].auprc
+
+        return stats