scihist 0.1.0__tar.gz → 0.1.9.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scihist-0.1.0 → scihist-0.1.9.dev0}/.gitignore +8 -2
- {scihist-0.1.0 → scihist-0.1.9.dev0}/PKG-INFO +2 -2
- {scihist-0.1.0 → scihist-0.1.9.dev0}/pyproject.toml +7 -3
- scihist-0.1.9.dev0/src/scihist/__init__.py +66 -0
- scihist-0.1.9.dev0/src/scihist/database.py +20 -0
- scihist-0.1.9.dev0/src/scihist/foreach.py +23 -0
- scihist-0.1.9.dev0/src/scihist/state.py +14 -0
- scihist-0.1.0/src/scihist/__init__.py +0 -68
- scihist-0.1.0/src/scihist/database.py +0 -51
- scihist-0.1.0/src/scihist/foreach.py +0 -797
- scihist-0.1.0/src/scihist/state.py +0 -917
- {scihist-0.1.0 → scihist-0.1.9.dev0}/README.md +0 -0
|
@@ -6,5 +6,11 @@ scistack-gui/extension/node_modules/
|
|
|
6
6
|
scistack-gui/frontend/node_modules/__pycache__/
|
|
7
7
|
*.pyc
|
|
8
8
|
__pycache__/
|
|
9
|
-
*.pyc
|
|
10
|
-
*.pyo
|
|
9
|
+
*.pyc
|
|
10
|
+
*.pyo
|
|
11
|
+
|
|
12
|
+
# Generated database artifacts (DuckDB data/lineage + write-ahead logs)
|
|
13
|
+
*.duckdb
|
|
14
|
+
*.duckdb.wal
|
|
15
|
+
*.wal
|
|
16
|
+
scistack-gui/frontend/node_modules/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scihist
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.9.dev0
|
|
4
4
|
Summary: Lineage-tracked batch execution for scientific data pipelines
|
|
5
5
|
Author: SciStack Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -17,8 +17,8 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
17
17
|
Classifier: Topic :: Scientific/Engineering
|
|
18
18
|
Classifier: Typing :: Typed
|
|
19
19
|
Requires-Python: >=3.9
|
|
20
|
-
Requires-Dist: scidb>=0.1.0
|
|
21
20
|
Requires-Dist: scilineage>=0.1.0
|
|
21
|
+
Requires-Dist: scistack-db>=0.1.0
|
|
22
22
|
Provides-Extra: dev
|
|
23
23
|
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
24
24
|
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["hatchling"]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
3
|
build-backend = "hatchling.build"
|
|
4
4
|
|
|
5
|
+
[tool.hatch.version]
|
|
6
|
+
source = "vcs"
|
|
7
|
+
raw-options = { search_parent_directories = true, local_scheme = "no-local-version" }
|
|
8
|
+
|
|
5
9
|
[project]
|
|
6
10
|
name = "scihist"
|
|
7
|
-
|
|
11
|
+
dynamic = ["version"]
|
|
8
12
|
description = "Lineage-tracked batch execution for scientific data pipelines"
|
|
9
13
|
readme = "README.md"
|
|
10
14
|
license = "MIT"
|
|
@@ -34,7 +38,7 @@ classifiers = [
|
|
|
34
38
|
"Typing :: Typed",
|
|
35
39
|
]
|
|
36
40
|
dependencies = [
|
|
37
|
-
"
|
|
41
|
+
"scistack-db>=0.1.0",
|
|
38
42
|
"scilineage>=0.1.0",
|
|
39
43
|
]
|
|
40
44
|
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""SciHist (deprecated) — thin shim over the consolidated scidb API.
|
|
2
|
+
|
|
3
|
+
Lineage-tracked batch execution, the node-staleness API, and lineage-aware
|
|
4
|
+
save now live in **scidb** (``scidb.for_each`` tracks lineage by default).
|
|
5
|
+
This package remains only as a backward-compatible shim for existing imports
|
|
6
|
+
(e.g. ``scistack-gui`` and the MATLAB ``+scihist`` bridge). Prefer importing
|
|
7
|
+
from ``scidb`` directly; ``scihist`` will be removed in a future release.
|
|
8
|
+
|
|
9
|
+
Behavioral nuances preserved by the shim:
|
|
10
|
+
- ``scihist.for_each`` defaults ``skip_computed=True`` (scidb defaults False).
|
|
11
|
+
- ``scihist.configure_database`` registers the DB as scilineage's cache backend.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import warnings as _warnings
|
|
15
|
+
|
|
16
|
+
# Core batch execution + lineage-aware save (shimmed to preserve scihist defaults)
|
|
17
|
+
# Re-export DB wrappers from scidb
|
|
18
|
+
# Re-export the step-function marker (replaces the removed @lineage_fcn).
|
|
19
|
+
from scidb import ColumnSelection, Fixed, ForEachConfig, Merge, scistack
|
|
20
|
+
|
|
21
|
+
# Re-export scifor helpers
|
|
22
|
+
from scifor import Col, PathInput, get_schema, set_schema
|
|
23
|
+
|
|
24
|
+
from .database import configure_database
|
|
25
|
+
from .foreach import for_each, save
|
|
26
|
+
from .state import (
|
|
27
|
+
check_combo_state,
|
|
28
|
+
check_multiple_nodes_state,
|
|
29
|
+
check_node_state,
|
|
30
|
+
check_pathinput_node_state,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
_warnings.warn(
|
|
34
|
+
"scihist is deprecated; its functionality has moved to scidb. "
|
|
35
|
+
"Import from scidb instead (e.g. `from scidb import for_each, save, "
|
|
36
|
+
"configure_database`).",
|
|
37
|
+
DeprecationWarning,
|
|
38
|
+
stacklevel=2,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
__version__ = "0.1.0"
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
# Core batch execution
|
|
45
|
+
"for_each",
|
|
46
|
+
"save",
|
|
47
|
+
# Configuration
|
|
48
|
+
"configure_database",
|
|
49
|
+
# Node staleness
|
|
50
|
+
"check_combo_state",
|
|
51
|
+
"check_node_state",
|
|
52
|
+
"check_pathinput_node_state",
|
|
53
|
+
"check_multiple_nodes_state",
|
|
54
|
+
# DB wrappers
|
|
55
|
+
"Fixed",
|
|
56
|
+
"Merge",
|
|
57
|
+
"ColumnSelection",
|
|
58
|
+
"ForEachConfig",
|
|
59
|
+
"PathInput",
|
|
60
|
+
# Schema helpers
|
|
61
|
+
"Col",
|
|
62
|
+
"set_schema",
|
|
63
|
+
"get_schema",
|
|
64
|
+
# Step-function marker
|
|
65
|
+
"scistack",
|
|
66
|
+
]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Deprecated shim — ``scihist.configure_database`` moved to scidb.
|
|
2
|
+
|
|
3
|
+
Prefer importing from ``scidb`` directly. This thin wrapper remains for
|
|
4
|
+
backward-compatible imports. (It previously also registered the database as
|
|
5
|
+
scilineage's cache backend; the rerun cache was removed, so it now simply
|
|
6
|
+
delegates to ``scidb.configure_database``.)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def configure_database(
|
|
13
|
+
db_path: str,
|
|
14
|
+
schema_keys: list[str] | None = None,
|
|
15
|
+
**kwargs,
|
|
16
|
+
) -> Any:
|
|
17
|
+
"""Deprecated alias for ``scidb.configure_database``."""
|
|
18
|
+
from scidb import configure_database as _scidb_configure
|
|
19
|
+
|
|
20
|
+
return _scidb_configure(db_path, schema_keys)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Deprecated shim — scihist.for_each / save moved into scidb.
|
|
2
|
+
|
|
3
|
+
The lineage-tracked batch execution that lived here is now part of
|
|
4
|
+
``scidb.for_each`` (which tracks lineage by default). This module remains so
|
|
5
|
+
existing ``from scihist.foreach import ...`` imports keep working; prefer
|
|
6
|
+
importing from ``scidb`` directly.
|
|
7
|
+
|
|
8
|
+
The one behavioral nuance preserved here is ``skip_computed=True`` as the
|
|
9
|
+
default (scidb.for_each defaults it to ``False``), matching the historical
|
|
10
|
+
scihist.for_each contract.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from scidb.foreach import for_each as _scidb_for_each
|
|
14
|
+
from scidb.lineage_save import save # noqa: F401 — re-exported for back-compat
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def for_each(*args, skip_computed: bool = True, **kwargs):
|
|
18
|
+
"""Deprecated alias for ``scidb.for_each`` with ``skip_computed=True``.
|
|
19
|
+
|
|
20
|
+
scidb.for_each tracks lineage by default; this wrapper only restores the
|
|
21
|
+
historical scihist default of skipping already-computed combos.
|
|
22
|
+
"""
|
|
23
|
+
return _scidb_for_each(*args, skip_computed=skip_computed, **kwargs)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Deprecated shim — scihist.state moved into scidb.state.
|
|
2
|
+
|
|
3
|
+
Re-exports the pipeline node staleness API from its new home in ``scidb.state``.
|
|
4
|
+
Prefer importing from ``scidb`` directly.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from scidb.state import ( # noqa: F401 — re-exported for back-compat
|
|
8
|
+
ComboState,
|
|
9
|
+
NodeState,
|
|
10
|
+
check_combo_state,
|
|
11
|
+
check_multiple_nodes_state,
|
|
12
|
+
check_node_state,
|
|
13
|
+
check_pathinput_node_state,
|
|
14
|
+
)
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
"""SciHist: Lineage-tracked batch execution for scientific data pipelines.
|
|
2
|
-
|
|
3
|
-
This package adds lineage tracking on top of scidb. It provides the same
|
|
4
|
-
for_each() interface as scidb, but automatically wraps functions in
|
|
5
|
-
LineageFcn for provenance recording.
|
|
6
|
-
|
|
7
|
-
Example:
|
|
8
|
-
from scihist import for_each, Fixed, configure_database
|
|
9
|
-
from scilineage import lineage_fcn
|
|
10
|
-
|
|
11
|
-
configure_database("experiment.duckdb", ["subject", "session"])
|
|
12
|
-
|
|
13
|
-
@lineage_fcn
|
|
14
|
-
def process_data(raw, calibration):
|
|
15
|
-
return raw * calibration
|
|
16
|
-
|
|
17
|
-
for_each(
|
|
18
|
-
process_data,
|
|
19
|
-
inputs={"raw": RawData, "calibration": Fixed(Calibration, session="baseline")},
|
|
20
|
-
outputs=[ProcessedData],
|
|
21
|
-
subject=[1, 2, 3],
|
|
22
|
-
session=["A", "B", "C"],
|
|
23
|
-
)
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
from .foreach import for_each, save
|
|
27
|
-
from .database import configure_database, find_by_lineage
|
|
28
|
-
from .state import check_combo_state, check_node_state, check_multiple_nodes_state
|
|
29
|
-
|
|
30
|
-
# Re-export DB wrappers from scidb
|
|
31
|
-
from scidb import Fixed, Merge, ColumnSelection, ForEachConfig
|
|
32
|
-
|
|
33
|
-
# Re-export scifor helpers
|
|
34
|
-
from scifor import Col, set_schema, get_schema, PathInput
|
|
35
|
-
|
|
36
|
-
# Re-export scilineage system
|
|
37
|
-
from scilineage import lineage_fcn, LineageFcn, LineageFcnResult, LineageFcnInvocation
|
|
38
|
-
|
|
39
|
-
__version__ = "0.1.0"
|
|
40
|
-
|
|
41
|
-
__all__ = [
|
|
42
|
-
# Core batch execution
|
|
43
|
-
"for_each",
|
|
44
|
-
"save",
|
|
45
|
-
# Configuration
|
|
46
|
-
"configure_database",
|
|
47
|
-
# Lineage query
|
|
48
|
-
"find_by_lineage",
|
|
49
|
-
# Node staleness
|
|
50
|
-
"check_combo_state",
|
|
51
|
-
"check_node_state",
|
|
52
|
-
"check_multiple_nodes_state",
|
|
53
|
-
# DB wrappers
|
|
54
|
-
"Fixed",
|
|
55
|
-
"Merge",
|
|
56
|
-
"ColumnSelection",
|
|
57
|
-
"ForEachConfig",
|
|
58
|
-
"PathInput",
|
|
59
|
-
# Schema helpers
|
|
60
|
-
"Col",
|
|
61
|
-
"set_schema",
|
|
62
|
-
"get_schema",
|
|
63
|
-
# Lineage system
|
|
64
|
-
"lineage_fcn",
|
|
65
|
-
"LineageFcn",
|
|
66
|
-
"LineageFcnResult",
|
|
67
|
-
"LineageFcnInvocation",
|
|
68
|
-
]
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
"""SciHist database configuration — wraps scidb.configure_database with cache backend registration."""
|
|
2
|
-
|
|
3
|
-
from typing import Any
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def configure_database(
|
|
7
|
-
db_path: str,
|
|
8
|
-
schema_keys: list[str] | None = None,
|
|
9
|
-
**kwargs,
|
|
10
|
-
) -> Any:
|
|
11
|
-
"""Configure the active database and register it as the lineage cache backend.
|
|
12
|
-
|
|
13
|
-
This is the scihist wrapper around scidb.configure_database(). In addition
|
|
14
|
-
to opening the DuckDB-backed database, it registers the database as the
|
|
15
|
-
cache backend so that lineage-tracked computations can look up previously
|
|
16
|
-
computed results.
|
|
17
|
-
|
|
18
|
-
Args:
|
|
19
|
-
db_path: Path to the DuckDB database file.
|
|
20
|
-
schema_keys: List of metadata keys that form the dataset schema.
|
|
21
|
-
**kwargs: Additional keyword arguments forwarded to scidb.configure_database().
|
|
22
|
-
|
|
23
|
-
Returns:
|
|
24
|
-
The configured DatabaseManager instance.
|
|
25
|
-
"""
|
|
26
|
-
from scidb import configure_database as _scidb_configure
|
|
27
|
-
from scilineage import configure_backend
|
|
28
|
-
|
|
29
|
-
db = _scidb_configure(db_path, schema_keys, **kwargs)
|
|
30
|
-
configure_backend(db)
|
|
31
|
-
return db
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def find_by_lineage(invocation) -> list | None:
|
|
35
|
-
"""Find output values by computation lineage.
|
|
36
|
-
|
|
37
|
-
Given a LineageFcnInvocation (function + inputs), finds any previously
|
|
38
|
-
computed outputs that match by querying the _lineage table via the active
|
|
39
|
-
database.
|
|
40
|
-
|
|
41
|
-
Args:
|
|
42
|
-
invocation: The LineageFcnInvocation computation to look up.
|
|
43
|
-
|
|
44
|
-
Returns:
|
|
45
|
-
List of output values if found, None otherwise.
|
|
46
|
-
"""
|
|
47
|
-
from scidb.database import get_database
|
|
48
|
-
|
|
49
|
-
db = get_database()
|
|
50
|
-
lineage_hash = invocation.compute_lineage_hash()
|
|
51
|
-
return db.find_by_lineage_hash(lineage_hash)
|