scilineage 0.1.0__tar.gz

scilineage-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.venv
+.DS_Store
+*/*/__pycache__
+others_projects/sciforge
+scistack-gui/extension/node_modules/
+scistack-gui/frontend/node_modules/__pycache__/ 
+*.pyc 
+__pycache__/ 
+*.pyc 
+*.pyo 

scilineage-0.1.0/PKG-INFO

@@ -0,0 +1,195 @@
+Metadata-Version: 2.4
+Name: scilineage
+Version: 0.1.0
+Summary: Lineage tracking for Python data pipelines
+Project-URL: Documentation, https://github.com/example/scilineage#readme
+Project-URL: Repository, https://github.com/example/scilineage
+Project-URL: Issues, https://github.com/example/scilineage/issues
+Author: SciStack Contributors
+License-Expression: MIT
+Keywords: data-science,lineage,pipeline,provenance,reproducibility
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: canonicalhash>=0.1.0
+Provides-Extra: all
+Requires-Dist: numpy>=1.20; extra == 'all'
+Requires-Dist: pandas>=1.3; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.0; extra == 'docs'
+Requires-Dist: mkdocs>=1.5; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
+Provides-Extra: numpy
+Requires-Dist: numpy>=1.20; extra == 'numpy'
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.3; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+# SciLineage
+
+**Lineage tracking for Python data pipelines.**
+
+SciLineage is a lightweight library for building data processing pipelines with automatic provenance tracking. It captures the full computational lineage of your results, enabling reproducibility and intelligent caching.
+
+## Features
+
+- **Automatic Lineage Tracking**: Every computation captures its inputs and function, building a complete provenance graph
+- **Input Classification**: Automatically distinguishes variable inputs from constants for accurate lineage
+- **Pluggable Caching**: Register a backend via `configure_backend()` to enable cache lookups via lineage hashes
+- **Lightweight**: Core dependency is only `canonicalhash`
+- **Type Safe**: Full type hints throughout
+
+## Installation
+
+```bash
+pip install scilineage
+```
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from scilineage import lineage_fcn
+
+@lineage_fcn
+def process(data, factor):
+    return data * factor
+
+# Call returns a LineageFcnResult, not the raw result
+result = process([1, 2, 3], 2)
+
+# Access the computed value
+print(result.data)  # [2, 4, 6]
+
+# Access lineage information
+print(result.invoked.inputs)         # {'arg_0': [1, 2, 3], 'arg_1': 2}
+print(result.invoked.fcn.fcn.__name__)  # 'process'
+```
+
+### Multi-Output Functions
+
+```python
+@lineage_fcn(unpack_output=True)
+def split_data(data):
+    mid = len(data) // 2
+    return data[:mid], data[mid:]
+
+first, second = split_data([1, 2, 3, 4])
+print(first.data)   # [1, 2]
+print(second.data)  # [3, 4]
+```
+
+### Chaining Computations
+
+```python
+@lineage_fcn
+def normalize(data):
+    max_val = max(data)
+    return [x / max_val for x in data]
+
+@lineage_fcn
+def scale(data, factor):
+    return [x * factor for x in data]
+
+raw = [10, 20, 30, 40]
+normalized = normalize(raw)
+scaled = scale(normalized, 100)
+
+print(scaled.data)  # [25.0, 50.0, 75.0, 100.0]
+```
+
+### Extracting Lineage
+
+```python
+from scilineage import extract_lineage, get_upstream_lineage
+
+@lineage_fcn
+def step1(x):
+    return x + 1
+
+@lineage_fcn
+def step2(x):
+    return x * 2
+
+result = step2(step1(5))
+
+lineage = extract_lineage(result)
+print(lineage.function_name)   # 'step2'
+print(lineage.function_hash)   # SHA-256 of function bytecode
+
+chain = get_upstream_lineage(result)
+for record in chain:
+    print(f"{record['function_name']}: inputs={record['inputs']}")
+```
+
+### Manual Interventions
+
+```python
+from scilineage import manual
+
+# Step outside the pipeline for a manual correction
+edited_data = [1, 2, 3]
+
+# Re-enter the pipeline — the intervention is documented in lineage
+corrected = manual(
+    edited_data,
+    label="outlier_removal",
+    reason="amplitude < 0.1 in trial 3 is sensor artifact",
+)
+```
+
+## API Reference
+
+### `@lineage_fcn(unpack_output=False, unwrap=True, generates_file=False)`
+
+Decorator to convert a function into a `LineageFcn`.
+
+- `unpack_output`: Whether to unpack a tuple return into separate `LineageFcnResult`s
+- `unwrap`: If True, automatically unwrap `LineageFcnResult` inputs to their raw data
+- `generates_file`: If True, marks the function as producing files as side effects
+
+### `LineageFcnResult`
+
+Wrapper around computed values that carries lineage.
+
+- `.data`: The actual computed value
+- `.invoked`: The `LineageFcnInvocation` that produced this
+- `.hash`: Unique hash based on computation lineage
+- `.output_num`: Index for multi-output functions
+
+### `LineageFcnInvocation`
+
+Represents a specific function invocation with captured inputs.
+
+- `.fcn`: The parent `LineageFcn` (function wrapper)
+- `.inputs`: Dict of captured input values
+- `.outputs`: Tuple of `LineageFcnResult` results
+- `.compute_lineage_hash()`: Generate lineage hash for cache key computation
+
+### `LineageFcn`
+
+The decorated function wrapper.
+
+- `.fcn`: The original wrapped function
+- `.hash`: SHA-256 hash of function bytecode
+- `.invocations`: All `LineageFcnInvocation`s created from this
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

scilineage-0.1.0/README.md

@@ -0,0 +1,153 @@
+# SciLineage
+
+**Lineage tracking for Python data pipelines.**
+
+SciLineage is a lightweight library for building data processing pipelines with automatic provenance tracking. It captures the full computational lineage of your results, enabling reproducibility and intelligent caching.
+
+## Features
+
+- **Automatic Lineage Tracking**: Every computation captures its inputs and function, building a complete provenance graph
+- **Input Classification**: Automatically distinguishes variable inputs from constants for accurate lineage
+- **Pluggable Caching**: Register a backend via `configure_backend()` to enable cache lookups via lineage hashes
+- **Lightweight**: Core dependency is only `canonicalhash`
+- **Type Safe**: Full type hints throughout
+
+## Installation
+
+```bash
+pip install scilineage
+```
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from scilineage import lineage_fcn
+
+@lineage_fcn
+def process(data, factor):
+    return data * factor
+
+# Call returns a LineageFcnResult, not the raw result
+result = process([1, 2, 3], 2)
+
+# Access the computed value
+print(result.data)  # [2, 4, 6]
+
+# Access lineage information
+print(result.invoked.inputs)         # {'arg_0': [1, 2, 3], 'arg_1': 2}
+print(result.invoked.fcn.fcn.__name__)  # 'process'
+```
+
+### Multi-Output Functions
+
+```python
+@lineage_fcn(unpack_output=True)
+def split_data(data):
+    mid = len(data) // 2
+    return data[:mid], data[mid:]
+
+first, second = split_data([1, 2, 3, 4])
+print(first.data)   # [1, 2]
+print(second.data)  # [3, 4]
+```
+
+### Chaining Computations
+
+```python
+@lineage_fcn
+def normalize(data):
+    max_val = max(data)
+    return [x / max_val for x in data]
+
+@lineage_fcn
+def scale(data, factor):
+    return [x * factor for x in data]
+
+raw = [10, 20, 30, 40]
+normalized = normalize(raw)
+scaled = scale(normalized, 100)
+
+print(scaled.data)  # [25.0, 50.0, 75.0, 100.0]
+```
+
+### Extracting Lineage
+
+```python
+from scilineage import extract_lineage, get_upstream_lineage
+
+@lineage_fcn
+def step1(x):
+    return x + 1
+
+@lineage_fcn
+def step2(x):
+    return x * 2
+
+result = step2(step1(5))
+
+lineage = extract_lineage(result)
+print(lineage.function_name)   # 'step2'
+print(lineage.function_hash)   # SHA-256 of function bytecode
+
+chain = get_upstream_lineage(result)
+for record in chain:
+    print(f"{record['function_name']}: inputs={record['inputs']}")
+```
+
+### Manual Interventions
+
+```python
+from scilineage import manual
+
+# Step outside the pipeline for a manual correction
+edited_data = [1, 2, 3]
+
+# Re-enter the pipeline — the intervention is documented in lineage
+corrected = manual(
+    edited_data,
+    label="outlier_removal",
+    reason="amplitude < 0.1 in trial 3 is sensor artifact",
+)
+```
+
+## API Reference
+
+### `@lineage_fcn(unpack_output=False, unwrap=True, generates_file=False)`
+
+Decorator to convert a function into a `LineageFcn`.
+
+- `unpack_output`: Whether to unpack a tuple return into separate `LineageFcnResult`s
+- `unwrap`: If True, automatically unwrap `LineageFcnResult` inputs to their raw data
+- `generates_file`: If True, marks the function as producing files as side effects
+
+### `LineageFcnResult`
+
+Wrapper around computed values that carries lineage.
+
+- `.data`: The actual computed value
+- `.invoked`: The `LineageFcnInvocation` that produced this
+- `.hash`: Unique hash based on computation lineage
+- `.output_num`: Index for multi-output functions
+
+### `LineageFcnInvocation`
+
+Represents a specific function invocation with captured inputs.
+
+- `.fcn`: The parent `LineageFcn` (function wrapper)
+- `.inputs`: Dict of captured input values
+- `.outputs`: Tuple of `LineageFcnResult` results
+- `.compute_lineage_hash()`: Generate lineage hash for cache key computation
+
+### `LineageFcn`
+
+The decorated function wrapper.
+
+- `.fcn`: The original wrapped function
+- `.hash`: SHA-256 hash of function bytecode
+- `.invocations`: All `LineageFcnInvocation`s created from this
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

scilineage-0.1.0/pyproject.toml

@@ -0,0 +1,98 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scilineage"
+version = "0.1.0"
+description = "Lineage tracking for Python data pipelines"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "SciStack Contributors" }
+]
+keywords = [
+    "lineage",
+    "provenance",
+    "pipeline",
+    "data-science",
+    "reproducibility",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+dependencies = [
+    "canonicalhash>=0.1.0",
+]
+
+[project.optional-dependencies]
+numpy = ["numpy>=1.20"]
+pandas = ["pandas>=1.3"]
+all = ["numpy>=1.20", "pandas>=1.3"]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "mypy>=1.0",
+    "ruff>=0.1.0",
+]
+docs = [
+    "mkdocs>=1.5",
+    "mkdocs-material>=9.0",
+    "mkdocstrings[python]>=0.24",
+]
+
+[project.urls]
+Documentation = "https://github.com/example/scilineage#readme"
+Repository = "https://github.com/example/scilineage"
+Issues = "https://github.com/example/scilineage/issues"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scilineage"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.ruff]
+target-version = "py310"
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # Pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+]
+ignore = [
+    "E501",  # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["scilineage"]

scilineage-0.1.0/src/scilineage/__init__.py

@@ -0,0 +1,77 @@
+"""SciLineage: Lineage Tracking for Python.
+
+A lightweight library for building data processing pipelines with automatic
+provenance tracking.
+
+Features:
+- Full lineage tracking for reproducibility
+- Automatic input capture and output wrapping
+- Lightweight (core dependency: canonicalhash)
+
+Example:
+    from scilineage import lineage_fcn
+
+    @lineage_fcn
+    def process(data, factor):
+        return data * factor
+
+    result = process(input_data, 2.5)  # Returns LineageFcnResult
+    print(result.data)  # The computed value
+    print(result.invoked.inputs)  # Captured inputs for provenance
+
+For multi-output functions, use unpack_output=True:
+
+    @lineage_fcn(unpack_output=True)
+    def split(data):
+        return data[:len(data)//2], data[len(data)//2:]
+
+    first_half, second_half = split(my_data)  # Each is a LineageFcnResult
+"""
+
+from .backend import configure_backend, _clear_backend
+from .core import (
+    LineageFcnResult,
+    LineageFcnInvocation,
+    LineageFcn,
+    lineage_fcn,
+    manual,
+    make_tuple_unpacking_wrapper,
+)
+from .hashing import canonical_hash, compute_function_hash
+from .inputs import InputKind, ClassifiedInput, classify_input, is_trackable_variable
+from .lineage import (
+    LineageRecord,
+    extract_lineage,
+    get_raw_value,
+    get_upstream_lineage,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Backend registry
+    "configure_backend",
+    # Core classes
+    "LineageFcn",
+    "LineageFcnInvocation",
+    "LineageFcnResult",
+    # Decorator
+    "lineage_fcn",
+    # Manual intervention
+    "manual",
+    # Tuple unpacking
+    "make_tuple_unpacking_wrapper",
+    # Input classification
+    "InputKind",
+    "ClassifiedInput",
+    "classify_input",
+    "is_trackable_variable",
+    # Lineage
+    "LineageRecord",
+    "extract_lineage",
+    "get_raw_value",
+    "get_upstream_lineage",
+    # Hashing
+    "canonical_hash",
+    "compute_function_hash",
+]

scilineage-0.1.0/src/scilineage/backend.py

@@ -0,0 +1,29 @@
+"""Backend registry for scilineage cache lookups.
+
+Provides a module-level slot for a single cache backend. Higher-level packages
+(scihist, scidb-net) register their database or network client here once at
+startup. End users do not interact with this directly.
+"""
+
+_backend = None
+
+
+def configure_backend(backend) -> None:
+    """Register a cache backend.
+
+    Called by scihist.configure_database() or scidb-net at startup.
+    Not intended to be called by end users directly.
+    """
+    global _backend
+    _backend = backend
+
+
+def _clear_backend() -> None:
+    """Reset the backend to None. For use in tests."""
+    global _backend
+    _backend = None
+
+
+def _get_backend():
+    """Return the current backend, or None if not configured."""
+    return _backend