deriva-ml 1.17.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deriva_ml/.DS_Store +0 -0
- deriva_ml/__init__.py +79 -0
- deriva_ml/bump_version.py +142 -0
- deriva_ml/core/__init__.py +39 -0
- deriva_ml/core/base.py +1527 -0
- deriva_ml/core/config.py +69 -0
- deriva_ml/core/constants.py +36 -0
- deriva_ml/core/definitions.py +74 -0
- deriva_ml/core/enums.py +222 -0
- deriva_ml/core/ermrest.py +288 -0
- deriva_ml/core/exceptions.py +28 -0
- deriva_ml/core/filespec.py +116 -0
- deriva_ml/dataset/__init__.py +12 -0
- deriva_ml/dataset/aux_classes.py +225 -0
- deriva_ml/dataset/dataset.py +1519 -0
- deriva_ml/dataset/dataset_bag.py +450 -0
- deriva_ml/dataset/history.py +109 -0
- deriva_ml/dataset/upload.py +439 -0
- deriva_ml/demo_catalog.py +495 -0
- deriva_ml/execution/__init__.py +26 -0
- deriva_ml/execution/environment.py +290 -0
- deriva_ml/execution/execution.py +1180 -0
- deriva_ml/execution/execution_configuration.py +147 -0
- deriva_ml/execution/workflow.py +413 -0
- deriva_ml/feature.py +228 -0
- deriva_ml/install_kernel.py +71 -0
- deriva_ml/model/__init__.py +0 -0
- deriva_ml/model/catalog.py +485 -0
- deriva_ml/model/database.py +719 -0
- deriva_ml/protocols/dataset.py +19 -0
- deriva_ml/run_notebook.py +228 -0
- deriva_ml/schema/__init__.py +3 -0
- deriva_ml/schema/annotations.py +473 -0
- deriva_ml/schema/check_schema.py +104 -0
- deriva_ml/schema/create_schema.py +393 -0
- deriva_ml/schema/deriva-ml-reference.json +8525 -0
- deriva_ml/schema/policy.json +81 -0
- deriva_ml/schema/table_comments_utils.py +57 -0
- deriva_ml/test.py +94 -0
- deriva_ml-1.17.10.dist-info/METADATA +38 -0
- deriva_ml-1.17.10.dist-info/RECORD +45 -0
- deriva_ml-1.17.10.dist-info/WHEEL +5 -0
- deriva_ml-1.17.10.dist-info/entry_points.txt +9 -0
- deriva_ml-1.17.10.dist-info/licenses/LICENSE +201 -0
- deriva_ml-1.17.10.dist-info/top_level.txt +1 -0
deriva_ml/.DS_Store
ADDED
|
Binary file
|
deriva_ml/__init__.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# We will be loading get_version from setuptools_scm and it will emit a UserWarning about it being deprecated.
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
# Safe imports - no circular dependencies
|
|
7
|
+
from deriva_ml.core.config import DerivaMLConfig
|
|
8
|
+
from deriva_ml.core.definitions import (
|
|
9
|
+
RID,
|
|
10
|
+
BuiltinTypes,
|
|
11
|
+
ColumnDefinition,
|
|
12
|
+
DerivaAssetColumns,
|
|
13
|
+
DerivaSystemColumns,
|
|
14
|
+
ExecAssetType,
|
|
15
|
+
ExecMetadataType,
|
|
16
|
+
FileSpec,
|
|
17
|
+
FileUploadState,
|
|
18
|
+
ForeignKeyDefinition,
|
|
19
|
+
KeyDefinition,
|
|
20
|
+
MLAsset,
|
|
21
|
+
MLVocab,
|
|
22
|
+
TableDefinition,
|
|
23
|
+
UploadState,
|
|
24
|
+
)
|
|
25
|
+
from deriva_ml.core.exceptions import (
|
|
26
|
+
DerivaMLException,
|
|
27
|
+
DerivaMLInvalidTerm,
|
|
28
|
+
DerivaMLTableTypeError,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Type-checking only - avoid circular import at runtime
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
from deriva_ml.core.base import DerivaML
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# Lazy import function for runtime usage
|
|
37
|
+
def __getattr__(name):
|
|
38
|
+
"""Lazy import to avoid circular dependencies."""
|
|
39
|
+
if name == "DerivaML":
|
|
40
|
+
from deriva_ml.core.base import DerivaML
|
|
41
|
+
|
|
42
|
+
return DerivaML
|
|
43
|
+
elif name == "Execution":
|
|
44
|
+
from deriva_ml.execution.execution import Execution
|
|
45
|
+
|
|
46
|
+
return Execution
|
|
47
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"DerivaML", # Lazy-loaded
|
|
52
|
+
"DerivaMLConfig",
|
|
53
|
+
# Exceptions
|
|
54
|
+
"DerivaMLException",
|
|
55
|
+
"DerivaMLInvalidTerm",
|
|
56
|
+
"DerivaMLTableTypeError",
|
|
57
|
+
# Definitions
|
|
58
|
+
"RID",
|
|
59
|
+
"BuiltinTypes",
|
|
60
|
+
"ColumnDefinition",
|
|
61
|
+
"DerivaSystemColumns",
|
|
62
|
+
"DerivaAssetColumns",
|
|
63
|
+
"ExecAssetType",
|
|
64
|
+
"ExecMetadataType",
|
|
65
|
+
"FileSpec",
|
|
66
|
+
"FileUploadState",
|
|
67
|
+
"ForeignKeyDefinition",
|
|
68
|
+
"KeyDefinition",
|
|
69
|
+
"MLAsset",
|
|
70
|
+
"MLVocab",
|
|
71
|
+
"TableDefinition",
|
|
72
|
+
"UploadState",
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
__version__ = version("deriva_ml")
|
|
77
|
+
except PackageNotFoundError:
|
|
78
|
+
# package is not installed
|
|
79
|
+
pass
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Release helper: seed initial semver tag if none exists, otherwise bump with bump-my-version.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
python release.py [patch|minor|major]
|
|
7
|
+
|
|
8
|
+
Env vars:
|
|
9
|
+
START - initial version if no tag exists (default: 0.1.0)
|
|
10
|
+
PREFIX - tag prefix (default: v)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Sequence
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def run(
|
|
24
|
+
cmd: Sequence[str], check: bool = True, capture: bool = False, quiet: bool = False
|
|
25
|
+
) -> subprocess.CompletedProcess:
|
|
26
|
+
if not quiet:
|
|
27
|
+
print(f"$ {' '.join(cmd)}")
|
|
28
|
+
return subprocess.run(
|
|
29
|
+
cmd,
|
|
30
|
+
check=check,
|
|
31
|
+
capture_output=capture,
|
|
32
|
+
text=True,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def in_git_repo() -> bool:
|
|
37
|
+
try:
|
|
38
|
+
run(["git", "rev-parse", "--is-inside-work-tree"], capture=True)
|
|
39
|
+
return True
|
|
40
|
+
except subprocess.CalledProcessError:
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def has_commits() -> bool:
|
|
45
|
+
try:
|
|
46
|
+
run(["git", "log", "-1"], capture=True)
|
|
47
|
+
return True
|
|
48
|
+
except subprocess.CalledProcessError:
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def latest_semver_tag(prefix: str) -> str | None:
|
|
53
|
+
# Use git's matcher to keep parity with Bash: prefix + x.y.z
|
|
54
|
+
pattern = f"{prefix}[0-9]*.[0-9]*.[0-9]*"
|
|
55
|
+
try:
|
|
56
|
+
cp = run(["git", "describe", "--tags", "--abbrev=0", "--match", pattern], capture=True, quiet=True)
|
|
57
|
+
tag = cp.stdout.strip()
|
|
58
|
+
return tag or None
|
|
59
|
+
except subprocess.CalledProcessError:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def seed_initial_tag(tag: str) -> None:
|
|
64
|
+
print(f"No existing semver tag found. Seeding initial tag: {tag}")
|
|
65
|
+
run(["git", "tag", tag, "-m", f"Initial release {tag}"])
|
|
66
|
+
# Push tags (ignore failure to keep parity with bash's simple flow)
|
|
67
|
+
run(["git", "push", "--tags"])
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def require_tool(name: str) -> None:
|
|
71
|
+
if shutil.which(name) is None:
|
|
72
|
+
print(f"Error: required tool '{name}' not found on PATH.", file=sys.stderr)
|
|
73
|
+
sys.exit(1)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def main() -> int:
|
|
77
|
+
parser = argparse.ArgumentParser(
|
|
78
|
+
description="Set a new version tag for the current repository, and push to remote."
|
|
79
|
+
)
|
|
80
|
+
parser.add_argument(
|
|
81
|
+
"bump", nargs="?", default="patch", choices=["patch", "minor", "major"], help="Which semver part to bump."
|
|
82
|
+
)
|
|
83
|
+
args = parser.parse_args()
|
|
84
|
+
|
|
85
|
+
start = os.environ.get("START", "0.1.0")
|
|
86
|
+
prefix = os.environ.get("PREFIX", "v")
|
|
87
|
+
|
|
88
|
+
# Sanity checks
|
|
89
|
+
require_tool("git")
|
|
90
|
+
require_tool("uv")
|
|
91
|
+
|
|
92
|
+
if not in_git_repo():
|
|
93
|
+
print("Not a git repo.", file=sys.stderr)
|
|
94
|
+
return 1
|
|
95
|
+
|
|
96
|
+
# Ensure tags visible in shallow clones
|
|
97
|
+
try:
|
|
98
|
+
run(["git", "fetch", "--tags", "--quiet"], check=False)
|
|
99
|
+
except Exception:
|
|
100
|
+
pass # non-fatal
|
|
101
|
+
|
|
102
|
+
if not has_commits():
|
|
103
|
+
print("No commits found. Commit something before tagging.", file=sys.stderr)
|
|
104
|
+
return 1
|
|
105
|
+
|
|
106
|
+
# Find latest semver tag with prefix
|
|
107
|
+
tag = latest_semver_tag(prefix)
|
|
108
|
+
print(f"Latest semver tag: {tag}")
|
|
109
|
+
if not tag:
|
|
110
|
+
seed_initial_tag(f"{prefix}{start}")
|
|
111
|
+
print(f"Seeded {prefix}{start}. Done.")
|
|
112
|
+
return 0
|
|
113
|
+
|
|
114
|
+
print(f"Bumping version: {args.bump}")
|
|
115
|
+
|
|
116
|
+
# Bump using bump-my-version via uv
|
|
117
|
+
# Mirrors: uv run bump-my-version bump $BUMP --verbose
|
|
118
|
+
try:
|
|
119
|
+
run(["uv", "run", "bump-my-version", "bump", args.bump, "--verbose"])
|
|
120
|
+
except subprocess.CalledProcessError as e:
|
|
121
|
+
print(e.stdout or "", end="")
|
|
122
|
+
print(e.stderr or "", end="", file=sys.stderr)
|
|
123
|
+
return e.returncode
|
|
124
|
+
|
|
125
|
+
# Push commits and tags
|
|
126
|
+
print("Pushing changes to remote repository...")
|
|
127
|
+
run(["git", "push", "--follow-tags"])
|
|
128
|
+
|
|
129
|
+
# Retrieve new version tag
|
|
130
|
+
try:
|
|
131
|
+
cp = run(["git", "describe", "--tags", "--abbrev=0"], capture=True)
|
|
132
|
+
new_tag = cp.stdout.strip()
|
|
133
|
+
print(f"New version tag: {new_tag}")
|
|
134
|
+
except subprocess.CalledProcessError:
|
|
135
|
+
print("Warning: unable to determine new tag via git describe.", file=sys.stderr)
|
|
136
|
+
|
|
137
|
+
print("Release process complete!")
|
|
138
|
+
return 0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
if __name__ == "__main__":
|
|
142
|
+
sys.exit(main())
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from deriva_ml.core.base import DerivaML
|
|
2
|
+
from deriva_ml.core.config import DerivaMLConfig
|
|
3
|
+
from deriva_ml.core.definitions import (
|
|
4
|
+
RID,
|
|
5
|
+
BuiltinTypes,
|
|
6
|
+
ColumnDefinition,
|
|
7
|
+
DerivaSystemColumns,
|
|
8
|
+
ExecAssetType,
|
|
9
|
+
ExecMetadataType,
|
|
10
|
+
FileSpec,
|
|
11
|
+
FileUploadState,
|
|
12
|
+
MLAsset,
|
|
13
|
+
MLVocab,
|
|
14
|
+
TableDefinition,
|
|
15
|
+
UploadState,
|
|
16
|
+
)
|
|
17
|
+
from deriva_ml.core.exceptions import DerivaMLException, DerivaMLInvalidTerm, DerivaMLTableTypeError
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"DerivaML",
|
|
21
|
+
"DerivaMLConfig",
|
|
22
|
+
# Exceptions
|
|
23
|
+
"DerivaMLException",
|
|
24
|
+
"DerivaMLInvalidTerm",
|
|
25
|
+
"DerivaMLTableTypeError",
|
|
26
|
+
# Definitions
|
|
27
|
+
"RID",
|
|
28
|
+
"BuiltinTypes",
|
|
29
|
+
"ColumnDefinition",
|
|
30
|
+
"DerivaSystemColumns",
|
|
31
|
+
"ExecAssetType",
|
|
32
|
+
"ExecMetadataType",
|
|
33
|
+
"FileSpec",
|
|
34
|
+
"FileUploadState",
|
|
35
|
+
"MLAsset",
|
|
36
|
+
"MLVocab",
|
|
37
|
+
"TableDefinition",
|
|
38
|
+
"UploadState",
|
|
39
|
+
]
|