weavehr 0.0.1__py3-none-any.whl

weavehr/__init__.py

@@ -0,0 +1,91 @@
+"""
+WeavEHR: A Python framework for extracting and analyzing ICU time series data.
+
+This package provides tools for extracting, preprocessing, and analyzing ICU time
+series data from diverse datasets (MIMIC, eICU, etc.). It converts heterogeneous
+ICU data into the standardized MEDS (Medical Event Data Standard) format, enabling
+reproducible research workflows.
+
+Key Features:
+- Support for multiple ICU data sources (MIMIC, eICU, custom datasets)
+- Declarative YAML-based configuration for data extraction
+- Export to MEDS format for standardized analysis
+- Fully offline operation for medical data privacy compliance
+- Modular and extensible architecture
+
+Main Modules:
+- callbacks: Data transformation callbacks for LazyFrames
+- config: Configuration management and registry system
+- storage: Project, dataset, and workspace management
+- steps: Processing steps (extraction, concept, sharding)
+- utils: Utility functions and helpers
+
+Usage:
+```python
+from pathlib import Path
+
+from weavehr import WeavEHRProject, ExtractionStep
+
+
+config_path = Path.cwd() / "config"
+project_path = Path.cwd() / "output" / "project"
+
+# Create a new WeavEHR project
+with WeavEHRProject(project_path) as project:
+    # Load and run an extraction step
+    extraction_step = ExtractionStep.load(project, config_path / "extraction.yml")
+    extraction_step.run()
+```
+
+Author:
+- Paul Brauckmann
+- Tobias Laser
+- Patrick Rockenschaub
+- Christian Porschen
+
+License:
+```
+MIT License
+
+Copyright (c) 2023 AIDH MS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+"""
+
+from weavehr.steps.concept.step import ConceptStep, concept_config_registry
+from weavehr.steps.extraction.step import ExtractionStep, dataset_config_registry
+from weavehr.steps.persistence.step import PersistenceStep
+from weavehr.steps.sharding.step import ShardingStep, sharding_config_registry
+from weavehr.storage.project import WeavEHRProject
+from weavehr.utils.loader import auto_load_configs
+
+auto_load_configs()
+
+__all__ = [
+    "WeavEHRProject",
+    "PersistenceStep",
+    "ExtractionStep",
+    "ConceptStep",
+    "ShardingStep",
+    "auto_load_configs",
+    "dataset_config_registry",
+    "concept_config_registry",
+    "sharding_config_registry",
+]

weavehr/callbacks/__init__.py

@@ -0,0 +1,73 @@
+from weavehr.callbacks._callbacks.algebra import (
+    Add,
+    Divide,
+    FloorDivide,
+    Modulo,
+    Multiply,
+    Pow,
+    Product,
+    Root,
+    Subtract,
+    Sum,
+)
+from weavehr.callbacks._callbacks.comparison import (
+    Equal,
+    GreaterEqual,
+    GreaterThan,
+    LessEqual,
+    LessThan,
+    NotEqual,
+)
+from weavehr.callbacks._callbacks.conditional import Replace
+from weavehr.callbacks._callbacks.filter import DropIf, DropNa, FirstDistinct
+from weavehr.callbacks._callbacks.logical import And, Not, Or
+from weavehr.callbacks._callbacks.reshape import SplitExplode
+from weavehr.callbacks._callbacks.selector import FirstNotNull, Max
+from weavehr.callbacks._callbacks.shortcuts import Col, Const
+from weavehr.callbacks._callbacks.string import ConcatStr, SliceStr, ZeroPadInt
+from weavehr.callbacks._callbacks.time import AddOffset, ParseDateTime, SetTime, ToDatetime
+from weavehr.callbacks._callbacks.type import Cast
+from weavehr.callbacks.proto import CallbackProtocol
+from weavehr.callbacks.registry import register_callback_cls, registry
+
+__all__ = [
+    "registry",
+    "register_callback_cls",
+    "CallbackProtocol",
+    "DropNa",
+    "FirstDistinct",
+    "DropIf",
+    "ParseDateTime",
+    "ToDatetime",
+    "AddOffset",
+    "SetTime",
+    "FirstNotNull",
+    "Max",
+    "Add",
+    "Sum",
+    "Subtract",
+    "Multiply",
+    "Product",
+    "Divide",
+    "FloorDivide",
+    "Pow",
+    "Root",
+    "Modulo",
+    "GreaterThan",
+    "LessThan",
+    "GreaterEqual",
+    "LessEqual",
+    "Equal",
+    "NotEqual",
+    "And",
+    "Or",
+    "Not",
+    "Col",
+    "Const",
+    "Replace",
+    "Cast",
+    "SplitExplode",
+    "ZeroPadInt",
+    "SliceStr",
+    "ConcatStr",
+]

weavehr/callbacks/_callbacks/algebra.py

@@ -0,0 +1,170 @@
+import operator
+from typing import Optional, Sequence
+
+import polars as pl
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_expr
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class Add(CallbackProtocol):
+    def __init__(self, augend: AstValue, addend: AstValue, output: Optional[str] = None) -> None:
+        self.augend = augend
+        self.addend = addend
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.augend) + to_expr(lf, self.addend)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Sum(CallbackProtocol):
+    def __init__(self, *summands: AstValue, output: Optional[str] = None) -> None:
+        if len(summands) == 1 and isinstance(summands[0], list):
+            self.summands: Sequence[AstValue] = summands[0]
+        else:
+            self.summands = summands
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        exprs = [to_expr(lf, s) for s in self.summands]
+
+        if not exprs:
+            expr = pl.lit(0)
+            if self.output is None:
+                return expr
+            return expr.alias(self.output)
+
+        expr = pl.sum_horizontal(exprs)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Subtract(CallbackProtocol):
+    def __init__(self, minuend: AstValue, subtrahend: AstValue, output: Optional[str] = None) -> None:
+        self.minuend = minuend
+        self.subtrahend = subtrahend
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.minuend) - to_expr(lf, self.subtrahend)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Multiply(CallbackProtocol):
+    def __init__(self, multiplicand: AstValue, multiplier: AstValue, output: Optional[str] = None) -> None:
+        self.multiplicand = multiplicand
+        self.multiplier = multiplier
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.multiplicand) * to_expr(lf, self.multiplier)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Product(CallbackProtocol):
+    def __init__(self, *factors: AstValue, output: Optional[str] = None) -> None:
+        if len(factors) == 1 and isinstance(factors[0], list):
+            self.factors: Sequence[AstValue] = factors[0]
+        else:
+            self.factors = factors
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        exprs = [to_expr(lf, f) for f in self.factors]
+
+        if not exprs:
+            expr = pl.lit(1)
+            if self.output is None:
+                return expr
+            return expr.alias(self.output)
+
+        expr = pl.fold(acc=pl.lit(1), function=operator.mul, exprs=exprs)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Divide(CallbackProtocol):
+    def __init__(self, dividend: AstValue, divisor: AstValue, output: Optional[str] = None) -> None:
+        self.dividend = dividend
+        self.divisor = divisor
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.dividend) / to_expr(lf, self.divisor)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class FloorDivide(CallbackProtocol):
+    def __init__(self, dividend: AstValue, divisor: AstValue, output: Optional[str] = None) -> None:
+        self.dividend = dividend
+        self.divisor = divisor
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.dividend) // to_expr(lf, self.divisor)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Pow(CallbackProtocol):
+    def __init__(self, base: AstValue, exponent: AstValue, output: Optional[str] = None) -> None:
+        self.base = base
+        self.exponent = exponent
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.base) ** to_expr(lf, self.exponent)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Root(CallbackProtocol):
+    def __init__(self, radicand: AstValue, index: AstValue, output: Optional[str] = None) -> None:
+        self.radicand = radicand
+        self.index = index
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        radicand_expr = to_expr(lf, self.radicand)
+        idx_expr = to_expr(lf, self.index)
+        expr = radicand_expr.sign() * radicand_expr.abs() ** (pl.lit(1) / idx_expr)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Modulo(CallbackProtocol):
+    def __init__(self, dividend: AstValue, divisor: AstValue, output: Optional[str] = None) -> None:
+        self.dividend = dividend
+        self.divisor = divisor
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = to_expr(lf, self.dividend) % to_expr(lf, self.divisor)
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)

weavehr/callbacks/_callbacks/comparison.py

@@ -0,0 +1,64 @@
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_expr
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class GreaterThan(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) > to_expr(lf, self.right)
+
+
+@register_callback_cls
+class LessThan(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) < to_expr(lf, self.right)
+
+
+@register_callback_cls
+class GreaterEqual(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) >= to_expr(lf, self.right)
+
+
+@register_callback_cls
+class LessEqual(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) <= to_expr(lf, self.right)
+
+
+@register_callback_cls
+class Equal(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) == to_expr(lf, self.right)
+
+
+@register_callback_cls
+class NotEqual(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) != to_expr(lf, self.right)

weavehr/callbacks/_callbacks/conditional.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+import polars as pl
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_expr
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class Replace(CallbackProtocol):
+    def __init__(
+        self,
+        condition: AstValue,
+        then_value: AstValue,
+        else_value: AstValue,
+        output: Optional[str] = None,
+    ) -> None:
+        self.condition = condition
+        self.then_value = then_value
+        self.else_value = else_value
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        expr = (
+            pl.when(to_expr(lf, self.condition))
+            .then(to_expr(lf, self.then_value))
+            .otherwise(to_expr(lf, self.else_value))
+        )
+
+        if self.output is None:
+            return expr
+
+        return expr.alias(self.output)

weavehr/callbacks/_callbacks/filter.py

@@ -0,0 +1,57 @@
+from typing import Sequence
+
+import polars as pl
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_col_name, to_expr
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class DropNa(CallbackProtocol):
+    def __init__(self, column: AstValue) -> None:
+        self.column = to_col_name(column)
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return pl.col(self.column).is_not_null()
+
+
+@register_callback_cls
+class FirstDistinct(CallbackProtocol):
+    """Keep the first row for each distinct combination of columns."""
+
+    def __init__(self, *columns: AstValue) -> None:
+        """
+        Initialize the callback with the column names used to identify duplicates.
+
+        Column names can be passed as separate arguments, for example:
+        ``first_distinct(subject_id, time)``.
+        """
+        if len(columns) == 1 and isinstance(columns[0], list):
+            self.columns: Sequence[AstValue] = columns[0]
+        else:
+            self.columns = columns
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        """Select the first row for each distinct combination of column values."""
+        column_names = [to_col_name(column) for column in self.columns]
+        return pl.struct(column_names).is_first_distinct()
+
+
+@register_callback_cls
+class DropIf(CallbackProtocol):
+    """
+    Drops rows where the given boolean condition is True.
+
+    Example:
+
+    Meaning:
+        keep rows where is_invalid is False
+    """
+
+    def __init__(self, condition: AstValue) -> None:
+        self.condition = condition
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        condition_expr = to_expr(lf, self.condition)
+        return condition_expr.not_()

weavehr/callbacks/_callbacks/logical.py

@@ -0,0 +1,33 @@
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_expr
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class And(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) & to_expr(lf, self.right)
+
+
+@register_callback_cls
+class Or(CallbackProtocol):
+    def __init__(self, left: AstValue, right: AstValue) -> None:
+        self.left = left
+        self.right = right
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return to_expr(lf, self.left) | to_expr(lf, self.right)
+
+
+@register_callback_cls
+class Not(CallbackProtocol):
+    def __init__(self, value: AstValue) -> None:
+        self.value = value
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        return ~to_expr(lf, self.value)

weavehr/callbacks/_callbacks/reshape.py

@@ -0,0 +1,28 @@
+from typing import Any
+
+import polars as pl
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, to_col_name
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class SplitExplode(CallbackProtocol):
+    def __init__(
+        self,
+        column: AstValue,
+        separator: str = ",",
+        strip: bool = True,
+    ) -> None:
+        self.column = to_col_name(column)
+        self.separator = separator
+        self.strip = strip
+
+    def __call__(self, lf: LazyFrame) -> Any:
+        lf = lf.with_columns(pl.col(self.column).str.split(self.separator).alias(self.column)).explode(self.column)
+
+        if self.strip:
+            lf = lf.with_columns(pl.col(self.column).str.strip_chars().alias(self.column))
+
+        return lf

weavehr/callbacks/_callbacks/selector.py

@@ -0,0 +1,113 @@
+from typing import Optional, Sequence
+
+import polars as pl
+from polars import LazyFrame
+
+from weavehr.callbacks.proto import AstValue, CallbackProtocol, CallbackResult, to_col_name
+from weavehr.callbacks.registry import register_callback_cls
+
+
+@register_callback_cls
+class FirstNotNull(CallbackProtocol):
+    """Create a column by selecting the first non-null value across columns.
+
+    This callback evaluates a list of source columns row-wise and assigns the
+    first non-null value (in order) to a new result column. If all specified
+    fields are null for a row, the resulting value is null. It is commonly used
+    for schema harmonization.
+    """
+
+    def __init__(self, *fields: AstValue, output: Optional[str]) -> None:
+        """Initialize the callback.
+
+        Args:
+            fields: Ordered list of source column names. Earlier columns
+                take precedence over later ones.
+            output: Name of the output column to be created.
+        """
+        if len(fields) == 1 and isinstance(fields[0], list):
+            self.fields: Sequence[AstValue] = fields[0]
+        else:
+            self.fields = fields
+
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        """Apply the transformation to a Polars LazyFrame.
+
+        Args:
+            lf: Input LazyFrame.
+
+        Returns:
+            A new LazyFrame with an additional column containing the first
+            non-null value per row across the specified fields.
+        """
+        cols = [to_col_name(f) for f in self.fields]
+
+        # Optional: empty list handling
+        if not cols:
+            # With no inputs, output is always null.
+            expr = pl.lit(None)
+            if self.output is None:
+                return expr
+            return expr.alias(self.output)
+
+        expr = pl.coalesce([pl.col(c) for c in cols])
+
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)
+
+
+@register_callback_cls
+class Max(CallbackProtocol):
+    """Create a column containing the maximum value across multiple columns.
+
+    This callback evaluates the specified source columns row-wise and returns
+    the maximum value for each row. Null values are ignored when at least one
+    non-null value is present. If all specified fields are null for a row, the
+    resulting value is null.
+    """
+
+    def __init__(self, *fields: AstValue, output: Optional[str]) -> None:
+        """Initialize the callback.
+
+        Args:
+            fields: Source column names whose row-wise maximum is selected.
+            output: Name of the output column. If `None`, the resulting
+                expression is returned without an alias.
+        """
+        if len(fields) == 1 and isinstance(fields[0], list):
+            self.fields: Sequence[AstValue] = fields[0]
+        else:
+            self.fields = fields
+
+        self.output = output
+
+    def __call__(self, lf: LazyFrame) -> CallbackResult:
+        """Create an expression for the row-wise maximum.
+
+        Args:
+            lf: Input LazyFrame.
+
+        Returns:
+            An expression containing the maximum value across the specified
+            fields for each row, optionally aliased with the output column name.
+            If no fields are specified, an expression containing null values is
+            returned.
+        """
+        cols = [to_col_name(f) for f in self.fields]
+
+        # Optional: empty list handling
+        if not cols:
+            # With no inputs, output is always null.
+            expr = pl.lit(None)
+            if self.output is None:
+                return expr
+            return expr.alias(self.output)
+
+        expr = pl.max_horizontal([pl.col(c) for c in cols])
+
+        if self.output is None:
+            return expr
+        return expr.alias(self.output)