scihist 0.1.0__tar.gz

scihist-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.venv
+.DS_Store
+*/*/__pycache__
+others_projects/sciforge
+scistack-gui/extension/node_modules/
+scistack-gui/frontend/node_modules/__pycache__/ 
+*.pyc 
+__pycache__/ 
+*.pyc 
+*.pyo 

scihist-0.1.0/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: scihist
+Version: 0.1.0
+Summary: Lineage-tracked batch execution for scientific data pipelines
+Author: SciStack Contributors
+License-Expression: MIT
+Keywords: automation,batch,data-science,lineage,pipeline,provenance
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: scidb>=0.1.0
+Requires-Dist: scilineage>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# SciHist
+
+Batch execution utilities for data pipelines.
+
+Provides utilities for running functions over combinations of metadata, automatically loading inputs and saving outputs.
+
+## Usage
+
+```python
+from scihist import for_each, Fixed
+
+for_each(
+    process_data,
+    inputs={"raw": RawData, "calibration": Fixed(Calibration, session="baseline")},
+    outputs=[ProcessedData],
+    subject=[1, 2, 3],
+    session=["A", "B", "C"],
+)
+```
+
+### `for_each`
+
+Executes a function for all combinations of metadata, loading inputs and saving outputs automatically.
+
+### `Fixed`
+
+Wrapper to specify fixed metadata overrides for an input. Use when an input should be loaded with different metadata than the current iteration.
+
+```python
+# Always load baseline from session="BL", regardless of current session
+for_each(
+    compare_to_baseline,
+    inputs={
+        "baseline": Fixed(StepLength, session="BL"),
+        "current": StepLength,
+    },
+    outputs=[Delta],
+    subject=[1, 2, 3],
+    session=["A", "B", "C"],
+)
+```

scihist-0.1.0/README.md

@@ -0,0 +1,41 @@
+# SciHist
+
+Batch execution utilities for data pipelines.
+
+Provides utilities for running functions over combinations of metadata, automatically loading inputs and saving outputs.
+
+## Usage
+
+```python
+from scihist import for_each, Fixed
+
+for_each(
+    process_data,
+    inputs={"raw": RawData, "calibration": Fixed(Calibration, session="baseline")},
+    outputs=[ProcessedData],
+    subject=[1, 2, 3],
+    session=["A", "B", "C"],
+)
+```
+
+### `for_each`
+
+Executes a function for all combinations of metadata, loading inputs and saving outputs automatically.
+
+### `Fixed`
+
+Wrapper to specify fixed metadata overrides for an input. Use when an input should be loaded with different metadata than the current iteration.
+
+```python
+# Always load baseline from session="BL", regardless of current session
+for_each(
+    compare_to_baseline,
+    inputs={
+        "baseline": Fixed(StepLength, session="BL"),
+        "current": StepLength,
+    },
+    outputs=[Delta],
+    subject=[1, 2, 3],
+    session=["A", "B", "C"],
+)
+```

scihist-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scihist"
+version = "0.1.0"
+description = "Lineage-tracked batch execution for scientific data pipelines"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "SciStack Contributors" }
+]
+keywords = [
+    "batch",
+    "pipeline",
+    "lineage",
+    "provenance",
+    "data-science",
+    "automation",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Typing :: Typed",
+]
+dependencies = [
+    "scidb>=0.1.0",
+    "scilineage>=0.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scihist"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

scihist-0.1.0/src/scihist/__init__.py

@@ -0,0 +1,68 @@
+"""SciHist: Lineage-tracked batch execution for scientific data pipelines.
+
+This package adds lineage tracking on top of scidb. It provides the same
+for_each() interface as scidb, but automatically wraps functions in
+LineageFcn for provenance recording.
+
+Example:
+    from scihist import for_each, Fixed, configure_database
+    from scilineage import lineage_fcn
+
+    configure_database("experiment.duckdb", ["subject", "session"])
+
+    @lineage_fcn
+    def process_data(raw, calibration):
+        return raw * calibration
+
+    for_each(
+        process_data,
+        inputs={"raw": RawData, "calibration": Fixed(Calibration, session="baseline")},
+        outputs=[ProcessedData],
+        subject=[1, 2, 3],
+        session=["A", "B", "C"],
+    )
+"""
+
+from .foreach import for_each, save
+from .database import configure_database, find_by_lineage
+from .state import check_combo_state, check_node_state, check_multiple_nodes_state
+
+# Re-export DB wrappers from scidb
+from scidb import Fixed, Merge, ColumnSelection, ForEachConfig
+
+# Re-export scifor helpers
+from scifor import Col, set_schema, get_schema, PathInput
+
+# Re-export scilineage system
+from scilineage import lineage_fcn, LineageFcn, LineageFcnResult, LineageFcnInvocation
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Core batch execution
+    "for_each",
+    "save",
+    # Configuration
+    "configure_database",
+    # Lineage query
+    "find_by_lineage",
+    # Node staleness
+    "check_combo_state",
+    "check_node_state",
+    "check_multiple_nodes_state",
+    # DB wrappers
+    "Fixed",
+    "Merge",
+    "ColumnSelection",
+    "ForEachConfig",
+    "PathInput",
+    # Schema helpers
+    "Col",
+    "set_schema",
+    "get_schema",
+    # Lineage system
+    "lineage_fcn",
+    "LineageFcn",
+    "LineageFcnResult",
+    "LineageFcnInvocation",
+]

scihist-0.1.0/src/scihist/database.py

@@ -0,0 +1,51 @@
+"""SciHist database configuration — wraps scidb.configure_database with cache backend registration."""
+
+from typing import Any
+
+
+def configure_database(
+    db_path: str,
+    schema_keys: list[str] | None = None,
+    **kwargs,
+) -> Any:
+    """Configure the active database and register it as the lineage cache backend.
+
+    This is the scihist wrapper around scidb.configure_database(). In addition
+    to opening the DuckDB-backed database, it registers the database as the
+    cache backend so that lineage-tracked computations can look up previously
+    computed results.
+
+    Args:
+        db_path: Path to the DuckDB database file.
+        schema_keys: List of metadata keys that form the dataset schema.
+        **kwargs: Additional keyword arguments forwarded to scidb.configure_database().
+
+    Returns:
+        The configured DatabaseManager instance.
+    """
+    from scidb import configure_database as _scidb_configure
+    from scilineage import configure_backend
+
+    db = _scidb_configure(db_path, schema_keys, **kwargs)
+    configure_backend(db)
+    return db
+
+
+def find_by_lineage(invocation) -> list | None:
+    """Find output values by computation lineage.
+
+    Given a LineageFcnInvocation (function + inputs), finds any previously
+    computed outputs that match by querying the _lineage table via the active
+    database.
+
+    Args:
+        invocation: The LineageFcnInvocation computation to look up.
+
+    Returns:
+        List of output values if found, None otherwise.
+    """
+    from scidb.database import get_database
+
+    db = get_database()
+    lineage_hash = invocation.compute_lineage_hash()
+    return db.find_by_lineage_hash(lineage_hash)