compehndly 0.0.1a1__tar.gz

compehndly-0.0.1a1/.gitignore

@@ -0,0 +1,140 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*venv/
+MANIFEST
+uv.lock
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+_notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Output folders
+*_output*/
+
+# Data
+_data/
+data/
+
+# temp
+_temp/
+
+# VScode stuff
+.vscode

compehndly-0.0.1a1/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: compehndly
+Version: 0.0.1a1
+Summary: Personal Exposure Health Registry
+Author-email: Gertjan Bisschop <gertjan.bisschop@vito.be>, Dirk Devriendt <dirk.devriendt@vito.be>, Caio Mescouto <caio.mescouto@vito.be>
+Requires-Python: <4.0.0,>=3.10
+Requires-Dist: pyarrow<23.0,>=22.0
+Requires-Dist: scipy
+Provides-Extra: numpy
+Requires-Dist: numpy>=2.0; extra == 'numpy'
+Provides-Extra: pandas
+Requires-Dist: pandas<3.0,>=2.0; extra == 'pandas'
+Provides-Extra: polars
+Requires-Dist: polars<2.0,>=1.0; extra == 'polars'

compehndly-0.0.1a1/pyproject.toml

@@ -0,0 +1,168 @@
+[project]
+name = "compehndly"
+version = "0.0.1a1"
+description = "Personal Exposure Health Registry"
+readme = "README.md"
+authors = [
+    { name = "Gertjan Bisschop", email = "gertjan.bisschop@vito.be" },
+    { name = "Dirk Devriendt", email = "dirk.devriendt@vito.be" },
+    { name = "Caio Mescouto", email = "caio.mescouto@vito.be" },
+]
+requires-python = ">=3.10, <4.0.0"
+dependencies = [
+	"pyarrow >= 22.0, <23.0",
+	"scipy",
+]
+
+[dependency-groups]
+test = [
+    "pytest>=8.2.0,<9",
+	"polars>=1.0, < 2.0",
+	"pandas>=2.0, < 3.0"
+]
+
+dev = [
+    "ruff>=0.4.4,<0.5",
+    "black",
+    "mypy",
+    "isort",
+    "pytest-cov>=7.0.0",
+]
+
+[tool.uv]
+default-groups = [
+    "test",
+    "dev",
+]
+
+[project.optional-dependencies]
+polars = [
+    "polars>=1.0,<2.0"
+]
+
+pandas = [
+    "pandas>=2.0,<3.0"
+]
+
+numpy = [
+    "numpy>=2.0"
+]
+
+[project.entry-points."compehndly.adapters"]
+polars = "compehndly.adapters.polars_adapter:PolarsAdapter"
+pandas = "compehndly.adapters.pandas_adapter:PandasAdapter"
+numpy = "compehndly.adapters.numpy_adapter:NumpyAdapter"
+
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/compehndly/",
+]
+
+[tool.hatch.build.targets.wheel]
+include = [
+    "src/compehndly/",
+	"src/compehndly/py.typed",
+]
+
+
+[tool.hatch.build.targets.wheel.sources]
+"src/compehndly" = "compehndly"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+pythonpath = [
+  "src"
+]
+addopts = [
+    "--import-mode=importlib",
+]
+markers = [
+	"base: base functionality",
+	"polars: polars dependency",
+	"numpy: numpy dependency",
+	"pandas: pandas dependency",
+]
+
+[tool.ruff]
+# Same as Black.
+line-length  = 120
+indent-width = 4
+
+# Assume Python 3.10
+target-version = "py310"
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+	".bzr",
+	".direnv",
+	".eggs",
+	".git",
+	".git-rewrite",
+	".hg",
+	".ipynb_checkpoints",
+	".mypy_cache",
+	".nox",
+	".pants.d",
+	".pyenv",
+	".pytest_cache",
+	".pytype",
+	".ruff_cache",
+	".svn",
+	".tox",
+	".venv",
+	".vscode",
+	"__pypackages__",
+	"_build",
+	"buck-out",
+	"build",
+	"dist",
+	"node_modules",
+	"site-packages",
+	"venv",
+]
+
+
+[tool.ruff.lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable   = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"

compehndly-0.0.1a1/src/compehndly/__init__.py

@@ -0,0 +1,70 @@
+import functools
+
+from compehndly.core.registry import FunctionRegistry
+from compehndly.adapters import register_all_adapters
+
+register_all_adapters()
+
+# Internal override hook
+_REGISTRY_BUILDER = None
+
+
+def _set_registry_builder(builder):
+    """
+    Test helper: override how the registry is built.
+    `builder` must be a callable returning a FunctionRegistry instance.
+    """
+    global _REGISTRY_BUILDER
+    _REGISTRY_BUILDER = builder
+    _REGISTRY.cache_clear()  # ensures next call uses new builder
+
+
+@functools.lru_cache
+def _REGISTRY():
+    """
+    Lazily build the registry, using the override when present.
+    """
+    if _REGISTRY_BUILDER is not None:
+        return _REGISTRY_BUILDER()
+
+    return FunctionRegistry.build_registry()
+
+
+class _FunctionAccessor:
+    """
+    Returned from compehndly.<func_name>
+    and provides:
+        compehndly.add_one(...)
+        compehndly.add_one["0.0.1"](...)
+    """
+
+    def __init__(self, name: str):
+        self._name = name
+
+    def __call__(self, *args, **kwargs):
+        """Invoke latest version."""
+        registry = _REGISTRY()
+        fn = registry.get(self._name, version=None)
+        return fn(*args, **kwargs)
+
+    def __getitem__(self, version: str | None):
+        """Invoke specific version (or None for latest)."""
+        registry = _REGISTRY()
+        fn = registry.get(self._name, version)
+        return fn
+
+
+def __getattr__(name: str):
+    """
+    Dynamically expose registered functions as attributes:
+        compehndly.add_one
+    """
+    registry = _REGISTRY()
+
+    if name in registry._functions:
+        return _FunctionAccessor(name)
+
+    raise AttributeError(f"'compehndly' has no function '{name}'")
+
+
+__all__ = ["FunctionRegistry"]

compehndly-0.0.1a1/src/compehndly/adapters/__init__.py

@@ -0,0 +1,25 @@
+import logging
+
+from importlib.metadata import entry_points
+from compehndly.adapters.base import ArrayAdapter
+
+logger = logging.getLogger(__name__)
+
+_ADAPTERS = {}
+
+
+def register_adapter(adapter: ArrayAdapter):
+    _ADAPTERS[adapter.name] = adapter
+
+
+def register_all_adapters():
+    logger.debug("Registering adapters")
+    for ep in entry_points(group="compehndly.adapters"):
+        try:
+            logger.debug(f"loading {ep}")
+            AdapterClass = ep.load()
+            register_adapter(AdapterClass())
+        except ImportError as e:
+            logger.warning(f"Failed to load {ep.name}: {e}")
+
+    register_adapter(ArrayAdapter())

compehndly-0.0.1a1/src/compehndly/adapters/base.py

@@ -0,0 +1,26 @@
+import logging
+import numbers
+import pyarrow as pa
+
+logger = logging.getLogger(__name__)
+
+
+class ArrayAdapter:
+    name: str = "base"
+
+    def _to_arrow(self, obj):
+        return pa.array(obj)
+
+    def to_arrow(self, obj) -> pa.Array:
+        if isinstance(obj, (pa.Array, pa.Scalar)):
+            return obj
+
+        # Python scalar → pass through unchanged
+        elif isinstance(obj, (numbers.Number, bool, str)):
+            return obj
+
+        # Python list/iterable → fallback to Arrow
+        return self._to_arrow(obj)
+
+    def from_arrow(self, obj):
+        return obj

compehndly-0.0.1a1/src/compehndly/adapters/numpy_adapter.py

@@ -0,0 +1,20 @@
+import logging
+import numpy as np
+import pyarrow as pa
+
+from compehndly.adapters.base import ArrayAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class NumpyAdapter(ArrayAdapter):
+    name = "numpy"
+
+    def matches(self, obj):
+        return isinstance(obj, np.ndarray) and obj.ndim == 1
+
+    def _to_arrow(self, obj):
+        return pa.array(obj)  # zero copy for many numeric dtypes
+
+    def from_arrow(self, arrow_obj):
+        return arrow_obj.to_numpy()  # zero-copy where possible

compehndly-0.0.1a1/src/compehndly/adapters/pandas_adapter.py

@@ -0,0 +1,20 @@
+import logging
+import pyarrow as pa
+import pandas as pd
+
+from compehndly.adapters.base import ArrayAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class PandasAdapter(ArrayAdapter):
+    name = "pandas"
+
+    def matches(self, obj):
+        return isinstance(obj, pd.Series)
+
+    def _to_arrow(self, obj):
+        return pa.array(obj)  # may be zero-copy if ArrowExtensionArray
+
+    def from_arrow(self, arrow_obj):
+        return pd.Series(arrow_obj)

compehndly-0.0.1a1/src/compehndly/adapters/polars_adapter.py

@@ -0,0 +1,22 @@
+import logging
+import polars as pl
+import pyarrow as pa
+
+from compehndly.adapters.base import ArrayAdapter
+
+logger = logging.getLogger(__name__)
+
+
+class PolarsAdapter(ArrayAdapter):
+    name = "polars"
+
+    def matches(self, obj):
+        return isinstance(obj, pl.Series)
+
+    def _to_arrow(self, obj):
+        if isinstance(obj, pl.Series):
+            return obj.to_arrow()
+        return pa.array(obj)
+
+    def from_arrow(self, arrow_obj):
+        return pl.from_arrow(arrow_obj)

compehndly-0.0.1a1/src/compehndly/core/__init__.py

@@ -0,0 +1,3 @@
+from compehndly.core.registry import FunctionRegistry
+
+__all__ = ["FunctionRegistry"]

compehndly-0.0.1a1/src/compehndly/core/conversion.py

@@ -0,0 +1,14 @@
+def arrowize_arguments(func, adapter):
+    def wrapper(*args, **kwargs):
+        # Convert args → arrow
+        arr_args = [adapter.to_arrow(a) for a in args]
+
+        # Convert kwargs → arrow (only values!)
+        arr_kwargs = {k: adapter.to_arrow(v) for k, v in kwargs.items()}
+
+        result = func(*arr_args, **arr_kwargs)
+
+        # Convert result → library-specific type
+        return adapter.from_arrow(result)
+
+    return wrapper

compehndly-0.0.1a1/src/compehndly/core/models.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class FunctionMetadata:
+    id: str
+    name: str
+    description: str
+    authors: list[str]

compehndly-0.0.1a1/src/compehndly/core/registry.py

@@ -0,0 +1,68 @@
+import importlib
+import logging
+
+from collections import defaultdict
+from packaging.version import Version
+
+from compehndly.core.conversion import arrowize_arguments
+from compehndly.adapters import _ADAPTERS
+
+logger = logging.getLogger(__name__)
+
+TO_REGISTER = ["compehndly.utils.example_function"]
+
+
+class FunctionRegistry:
+    def __init__(self, adapter: str | None = None):
+        self._functions = defaultdict(dict)
+        logging.debug("Running function registry")
+        if adapter is None:
+            self.adapter = _ADAPTERS["base"]
+        else:
+            if adapter not in _ADAPTERS:
+                raise ValueError(f"Unknown adapter '{adapter}'. " f"Available: {', '.join(_ADAPTERS)}")
+            self.adapter = _ADAPTERS[adapter]
+
+    def register(self, name, version, func):
+        version = Version(version)
+        if version in self._functions[name]:
+            raise ValueError(f"Function {name} version {version} already registered.")
+
+        wrapped_func = arrowize_arguments(func, adapter=self.adapter)
+        self._functions[name][version] = wrapped_func
+
+    def get(self, name, version=None):
+        """Return the function. If version is None, return latest."""
+        if name not in self._functions:
+            raise KeyError(f"No function registered with name '{name}'")
+        versions = sorted(self._functions[name].keys())
+        if version is None:
+            version = versions[-1]
+        else:
+            version = Version(version)
+        return self._functions[name][version]
+
+    def list_versions(self, name):
+        if name not in self._functions:
+            return []
+        return sorted(str(v) for v in self._functions[name].keys())
+
+    @classmethod
+    def build_registry(cls, _to_register=TO_REGISTER, adapter: str | None = None):
+        registry = cls(adapter=adapter)
+
+        for module_path in _to_register:
+            try:
+                module = importlib.import_module(module_path)
+            except ImportError as e:
+                raise ImportError(f"Failed to import module '{module_path}': {e}")
+
+            if not hasattr(module, "__registrations__") or not isinstance(module.__registrations__, list):
+                continue
+
+            for registry_name, func_name, version, func in module.__registrations__:
+                if registry_name != "default":
+                    raise ValueError(f"Unsupported registry name '{registry_name}' in module '{module_path}'")
+                registry.register(func_name, version, func)
+
+        return registry

compehndly-0.0.1a1/src/compehndly/core/registrymanager.py

@@ -0,0 +1 @@
+# TODO: allow support for multiple registries

compehndly-0.0.1a1/src/compehndly/derived_variables/correction.py

@@ -0,0 +1,96 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+__registrations__ = []
+
+
+# TODO: move decorator for joint use
+def register(registry_name, name, version):
+    def decorator(func):
+        __registrations__.append((registry_name, name, version, func))
+        return func
+
+    return decorator
+
+
+# ---------------- STANDARDIZE ----------------------
+
+
+def _standardize_v0_0_1_reference(
+    measured: float,
+    standard: float,
+) -> float:
+    return 100 * measured / standard
+
+
+@register(registry_name="default", name="standardize", version="0.0.1")
+def _standardize_v0_0_1_arrow(measured: pa.Array, standard: pa.Array) -> pa.Array:
+    return pc.divide(pc.multiply(measured, 100), standard)
+
+
+# ---------------- URINARY BIOMARKERS ----------------------
+
+
+def _standardize_creatinine_v0_0_1_reference(
+    measured: float,
+    crt: float,
+) -> float:
+    "measured in micrograms/L, crt in mg/dL"
+    return _standardize_v0_0_1_reference(measured, crt)
+
+
+@register(registry_name="default", name="standardize_creatinine", version="0.0.1")
+def _standardize_creatinine_v0_0_1_arrow(measured: pa.Array, crt: pa.Array) -> pa.Array:
+    return _standardize_v0_0_1_arrow(measured, crt)
+
+
+def _normalize_specific_gravity_v0_0_1_reference(
+    measured: float,
+    sg_measured: float,
+    sg_ref: float,
+) -> float:
+    ret = measured * (sg_ref - 1) / sg_measured
+
+    return ret
+
+
+@register(registry_name="default", name="normalize_specific_gravity", version="0.0.1")
+def _normalize_specific_gravity_v0_0_1_arrow(measured: pa.Array, sg_measured: pa.Array, sg_ref: float) -> pa.Array:
+    # Compute (sg_ref - 1) as a scalar
+    sg_factor = pa.scalar(sg_ref - 1, type=pa.float64())
+
+    # measured * (sg_ref - 1)
+    numerator = pc.multiply(measured, sg_factor)
+
+    # (measured * (sg_ref - 1)) / sg_measured
+    ret = pc.divide(numerator, sg_measured)
+
+    return ret
+
+
+# ---------------- LIPID SOLUBLE BIOMARKERS ----------------------
+
+
+def _total_lipid_concentration_v0_0_1_reference(chol: float, trigl: float) -> float:
+    """The total blood lipid content was calculated using the formula proposed by
+    (Bernert et al., 2007; Phillips et al., 1989) as advised by the analytical experts of HBM4EU.
+    Total lipids (mg/dL) = 2.27 * cholesterol (mg/dL) + triglycerides (mg/dL) + 62.3
+    """
+    return 2.27 * chol + trigl + 62.3
+
+
+@register(registry_name="default", name="total_lipid_concentration", version="0.0.1")
+def _total_lipid_concentration_v0_0_1_arrow(chol: pa.Array, trigl: pa.Array) -> pa.Array:
+    return pc.add(pc.multiply(chol, 2.27), pc.add(trigl, 62.3))
+
+
+def _standardize_lipid_v0_0_1_reference(
+    measured: float,
+    lipid_value: float,
+) -> float:
+    return _standardize_v0_0_1_reference(measured, lipid_value)
+
+
+@register(registry_name="default", name="normalize_specific_gravity", version="0.0.1")
+def _standardize_lipid_v0_0_1_arrow(measured: pa.Array, lipid_value: pa.Array) -> pa.Array:
+    return _standardize_v0_0_1_arrow(measured, lipid_value)

compehndly-0.0.1a1/src/compehndly/derived_variables/imputation.py

@@ -0,0 +1,195 @@
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from compehndly.derived_variables.statsutils import fit_censored_lognorm
+
+__registrations__ = []
+
+
+# TODO: move decorator for joint use
+def register(registry_name, name, version):
+    def decorator(func):
+        __registrations__.append((registry_name, name, version, func))
+        return func
+
+    return decorator
+
+
+def _medium_bound_imputation_v0_0_1_reference(
+    measurement: float,
+    loq: float,
+    lod: float | None = None,
+) -> float:
+    if lod is not None:
+        assert lod > 0.0
+    assert loq > 0.0
+    ret = measurement
+
+    if lod is None:
+        if measurement < loq:
+            ret = loq / 2
+    else:
+        assert lod < loq
+        if measurement < lod:
+            ret = lod / 2
+        elif measurement < loq:
+            ret = (loq + lod) / 2
+
+    return ret
+
+
+@register(registry_name="default", name="medium_bound_imputation", version="0.0.1")
+def _medium_bound_imputation_v0_0_1_arrow(
+    measurement: pa.Array,
+    loq: float,
+    lod: float | None = None,
+) -> pa.Array:
+    """
+    Vectorized medium-bound imputation.
+
+    """
+
+    if loq <= 0:
+        raise ValueError("loq must be > 0")
+
+    if lod is not None:
+        if lod <= 0:
+            raise ValueError("lod must be > 0")
+        if lod >= loq:
+            raise ValueError("lod must be < loq")
+
+    # Start with identity (original measurement)
+    result = measurement
+
+    if lod is None:
+        # measurement < loq → loq / 2
+        mask = pc.less(measurement, loq)
+        result = pc.if_else(mask, loq / 2, result)
+
+    else:
+        # measurement < lod → lod / 2
+        mask_below_lod = pc.less(measurement, lod)
+        result = pc.if_else(mask_below_lod, lod / 2, result)
+
+        # lod <= measurement < loq → (lod + loq) / 2
+        mask_between = pc.and_(
+            pc.greater_equal(measurement, lod),
+            pc.less(measurement, loq),
+        )
+        result = pc.if_else(mask_between, (lod + loq) / 2, result)
+
+    return result
+
+
+@register(registry_name="default", name="medium_bound_imputation_array", version="0.0.1")
+def _medium_bound_imputation_v0_0_1_arrow_array(
+    measurement: pa.Array,
+    loq: pa.Array,
+    lod: pa.Array | None = None,
+) -> pa.Array:
+    """
+    Vectorized medium-bound imputation with row-wise LOQ / LOD arrays.
+    """
+
+    length = len(measurement)
+
+    if len(loq) != length:
+        raise ValueError("measurement and loq must have the same length")
+
+    if lod is not None and len(lod) != length:
+        raise ValueError("measurement and lod must have the same length")
+
+    result = measurement
+
+    if lod is None:
+        # measurement < loq → loq / 2
+        mask = pc.less(measurement, loq)
+        result = pc.if_else(mask, pc.divide(loq, 2.0), result)
+
+    else:
+        # measurement < lod → lod / 2
+        mask_below_lod = pc.less(measurement, lod)
+        result = pc.if_else(mask_below_lod, pc.divide(lod, 2.0), result)
+
+        # lod <= measurement < loq → (lod + loq) / 2
+        mask_between = pc.and_(
+            pc.greater_equal(measurement, lod),
+            pc.less(measurement, loq),
+        )
+        midpoint = pc.divide(pc.add(lod, loq), 2.0)
+        result = pc.if_else(mask_between, midpoint, result)
+
+    return result
+
+
+def _random_single_imputation_reference_v0_0_1(
+    measurement: float,
+    loq: float,
+    lod: float | None = None,
+) -> float:
+    # TODO:
+    pass
+
+
+@register(registry_name="default", name="random_single_imputation", version="0.0.1")
+def _random_single_imputation_arrow_v0_0_1(
+    biomarker_pa: pa.Array,
+    lod: float,
+    loq: float,
+    seed: int | None = None,
+) -> pa.Array:
+    """
+    Perform random single imputation for left-censored lognormal data
+    using PyArrow arrays for maximum compatibility.
+
+    biomarker_pa : arrow array of floats or censored indicators (-1, -2, -3)
+    lod       : limit of detection
+    loq       : limit of quantification
+    """
+
+    # Convert Arrow → NumPy
+    # NOTE: suboptimal solution
+    biomarker = biomarker_pa.to_numpy(zero_copy_only=False)
+
+    # Fill NA as -1 (your original convention)
+    is_na = np.isnan(biomarker)
+    biomarker_filled = np.where(is_na, -1, biomarker)
+
+    # Censored if negative category code
+    censored = biomarker_filled < 0
+    values_np = np.where(censored, lod, biomarker_filled)
+    dist = fit_censored_lognorm(values_np, censored)
+    rng = np.random.default_rng(seed=seed)
+
+    # Compute sampling bounds (vectorized)
+    lower = np.zeros_like(biomarker_filled, dtype=float)
+    upper = np.zeros_like(biomarker_filled, dtype=float)
+
+    cat_below_lod = biomarker_filled == -1
+    cat_between = biomarker_filled == -2
+    cat_below_loq = biomarker_filled == -3
+
+    # <LOD -> [0, LOD]
+    lower[cat_below_lod] = 0
+    upper[cat_below_lod] = lod
+    # Between LOD & LOQ -> [LOD, LOQ]
+    lower[cat_between] = lod
+    upper[cat_between] = loq
+
+    # <LOQ -> [0, LOQ]
+    lower[cat_below_loq] = 0
+    upper[cat_below_loq] = loq
+
+    # Convert bounds to CDF space
+    cdf_lo = dist.cdf(lower)
+    cdf_hi = dist.cdf(upper)
+
+    # generate U ~ Uniform(cdf_lo, cdf_hi)
+    u = rng.uniform(cdf_lo, cdf_hi)
+    imputed = dist.ppf(u)
+    # replace censored with imputed
+    result = biomarker.copy()
+    result[censored] = imputed[censored]
+
+    return pa.array(result)

compehndly-0.0.1a1/src/compehndly/derived_variables/statsutils.py

@@ -0,0 +1,50 @@
+import numpy as np
+
+from scipy.stats import lognorm
+from scipy.optimize import minimize
+
+
+def fit_censored_lognorm(values_np, censored_np):
+    vals = values_np
+    cens = censored_np
+
+    # Require at least some uncensored observations
+    if (~cens).sum() == 0:
+        raise RuntimeError("Cannot fit lognormal: all observations are censored.")
+
+    def nll(params):
+        sigma, mu = params
+        if sigma <= 0:
+            return np.inf
+
+        dist = lognorm(s=sigma, scale=np.exp(mu))
+        ll_unc = dist.logpdf(vals[~cens]).sum()
+        ll_cens = np.log(dist.cdf(vals[cens])).sum()
+        ll = ll_unc + ll_cens
+
+        # Regularization to avoid pathological sigma
+        penalty = 0
+        if sigma < 0.05:
+            penalty += 1e3 * (0.05 - sigma) ** 2
+        if sigma > 5.0:
+            penalty += 1e3 * (sigma - 5.0) ** 2
+
+        return -(ll - penalty)
+
+    unc = vals[~cens]
+    mu0 = np.log(np.median(unc))
+    sigma0 = np.std(np.log(unc))
+    sigma0 = sigma0 if sigma0 > 0.1 else 0.5  # stability
+
+    res = minimize(
+        nll,
+        x0=[sigma0, mu0],
+        method="L-BFGS-B",
+        bounds=[(1e-6, None), (None, None)],
+    )
+
+    if not res.success:
+        raise RuntimeError(f"Censored MLE did not converge: {res.message}")
+
+    sigma_hat, mu_hat = res.x
+    return lognorm(s=sigma_hat, scale=np.exp(mu_hat))

compehndly-0.0.1a1/src/compehndly/derived_variables/summation.py

@@ -0,0 +1,63 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+
+__registrations__ = []
+
+
+# TODO: move decorator for joint use
+def register(registry_name, name, version):
+    def decorator(func):
+        __registrations__.append((registry_name, name, version, func))
+        return func
+
+    return decorator
+
+
+def _summation_v0_0_1_reference(
+    *arrays: float,
+    all_required=True,
+) -> float | None:
+    if not arrays:
+        raise ValueError("At least one input array is required")
+
+    if all_required:
+        if not any(x is not None for x in arrays):
+            return None
+
+    ret = 0
+    for val in arrays:
+        if val is not None:
+            ret += val
+
+    return ret
+
+
+@register(registry_name="default", name="summation", version="0.0.1")
+def _summation_v0_0_1_arrow(*arrays: pa.Array, all_required=True) -> pa.Array:
+    """
+    Vectorized summation over multiple Arrow arrays.
+
+    Semantics:
+    - If ANY input array is entirely null, return an all-null array.
+    - Otherwise, nulls are treated as zero during summation.
+    """
+
+    if not arrays:
+        raise ValueError("At least one input array is required")
+
+    length = len(arrays[0])
+
+    for arr in arrays:
+        if len(arr) != length:
+            raise ValueError("All input arrays must have the same length")
+
+        if arr.null_count == length and all_required:
+            return pa.nulls(length, type=pa.float64())
+
+    result = pc.fill_null(arrays[0], 0)
+
+    for arr in arrays[1:]:
+        result = pc.add(result, pc.fill_null(arr, 0))
+
+    return result

compehndly-0.0.1a1/src/compehndly/utils/bins.py

@@ -0,0 +1,26 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+__registrations__ = []
+
+
+# TODO: move decorator for joint use
+def register(registry_name, name, version):
+    def decorator(func):
+        __registrations__.append((registry_name, name, version, func))
+        return func
+
+    return decorator
+
+
+def _simple_bin_v0_0_1_reference(
+    *arrays: float,
+) -> float:
+    pass
+
+
+@register(registry_name="default", name="summation", version="0.0.1")
+def _simple_bin_v0_0_1_arrow(
+    array: pa.Array,
+) -> pa.Array:
+    pass

compehndly-0.0.1a1/src/compehndly/utils/conditionals.py

@@ -0,0 +1,16 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+__registrations__ = []
+
+
+# TODO: move decorator for joint use
+def register(registry_name, name, version):
+    def decorator(func):
+        __registrations__.append((registry_name, name, version, func))
+        return func
+
+    return decorator
+
+
+# TODO: add conditional helper functions