sim-atlas-toolkit 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sim_atlas_toolkit/__init__.py +3 -0
- sim_atlas_toolkit/cli.py +99 -0
- sim_atlas_toolkit/models.py +112 -0
- sim_atlas_toolkit/node_store.py +187 -0
- sim_atlas_toolkit/parser.py +30 -0
- sim_atlas_toolkit/parsers/dataclass_node.py +93 -0
- sim_atlas_toolkit/parsers/metadata.py +82 -0
- sim_atlas_toolkit/parsers/pyiron_core.py +35 -0
- sim_atlas_toolkit/parsers/pyiron_workflow.py +49 -0
- sim_atlas_toolkit/parsers/python_function.py +69 -0
- sim_atlas_toolkit/parsers/python_workflow_definition.py +45 -0
- sim_atlas_toolkit-0.0.1.dist-info/METADATA +27 -0
- sim_atlas_toolkit-0.0.1.dist-info/RECORD +16 -0
- sim_atlas_toolkit-0.0.1.dist-info/WHEEL +4 -0
- sim_atlas_toolkit-0.0.1.dist-info/entry_points.txt +3 -0
- sim_atlas_toolkit-0.0.1.dist-info/licenses/LICENSE +28 -0
sim_atlas_toolkit/cli.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import importlib
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
from .node_store import NodeStore
|
|
9
|
+
|
|
10
|
+
DEFAULT_API_URL_ENV = "SIM_ATLAS_API_URL"
|
|
11
|
+
DEFAULT_API_TOKEN_ENV = "SIM_ATLAS_API_TOKEN"
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("SimAtlas")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="sim-atlas-upload",
|
|
19
|
+
description=(
|
|
20
|
+
"Upload one or more Python modules to a Sim Atlas backend. "
|
|
21
|
+
"Values can be provided via CLI flags or environment variables."
|
|
22
|
+
),
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"modules",
|
|
26
|
+
nargs="+",
|
|
27
|
+
help="Module name(s) to upload, for example 'mypackage.mymodule'.",
|
|
28
|
+
)
|
|
29
|
+
parser.add_argument(
|
|
30
|
+
"--api-url",
|
|
31
|
+
default=os.getenv(DEFAULT_API_URL_ENV),
|
|
32
|
+
help=(f"Backend API base URL. Defaults to ${DEFAULT_API_URL_ENV} if omitted."),
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--api-token",
|
|
36
|
+
default=os.getenv(DEFAULT_API_TOKEN_ENV),
|
|
37
|
+
help=(
|
|
38
|
+
f"API token sent as x-api-key. Defaults to ${DEFAULT_API_TOKEN_ENV} "
|
|
39
|
+
"if omitted."
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"--recursive",
|
|
44
|
+
choices=["no", "import", "filesystem"],
|
|
45
|
+
default="no",
|
|
46
|
+
help="Recursion strategy for module traversal.",
|
|
47
|
+
)
|
|
48
|
+
parser.add_argument(
|
|
49
|
+
"--update-existing",
|
|
50
|
+
action="store_true",
|
|
51
|
+
help="Update existing nodes if they already exist.",
|
|
52
|
+
)
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--verbose",
|
|
55
|
+
action="store_true",
|
|
56
|
+
help="Enable verbose logging.",
|
|
57
|
+
)
|
|
58
|
+
return parser
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def main() -> int:
|
|
62
|
+
parser = _build_parser()
|
|
63
|
+
args = parser.parse_args()
|
|
64
|
+
|
|
65
|
+
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
|
66
|
+
|
|
67
|
+
if not args.api_url:
|
|
68
|
+
parser.error(
|
|
69
|
+
f"Missing API URL. Provide --api-url or set {DEFAULT_API_URL_ENV}."
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Fail fast on import errors so CLI can return a non-zero exit code.
|
|
73
|
+
missing_modules: list[str] = []
|
|
74
|
+
for module_name in args.modules:
|
|
75
|
+
try:
|
|
76
|
+
importlib.import_module(module_name)
|
|
77
|
+
except Exception as exc: # noqa: BLE001
|
|
78
|
+
missing_modules.append(module_name)
|
|
79
|
+
logger.error("Failed to import module %s: %s", module_name, exc)
|
|
80
|
+
|
|
81
|
+
if missing_modules:
|
|
82
|
+
logger.error(
|
|
83
|
+
"Aborting upload because one or more modules could not be imported."
|
|
84
|
+
)
|
|
85
|
+
return 1
|
|
86
|
+
|
|
87
|
+
store = NodeStore(api_url=args.api_url, api_key=args.api_token)
|
|
88
|
+
for module_name in args.modules:
|
|
89
|
+
store.upload_module(
|
|
90
|
+
module_name,
|
|
91
|
+
update_existing=args.update_existing,
|
|
92
|
+
recursive=args.recursive,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__":
|
|
99
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ArtifactType(StrEnum):
|
|
7
|
+
FUNCTION = "function"
|
|
8
|
+
WORKFLOW = "workflow"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Annotation(BaseModel):
|
|
12
|
+
has_default_value: bool = False
|
|
13
|
+
label: str | None = None
|
|
14
|
+
datatype: str | None = None
|
|
15
|
+
unit: str | None = None
|
|
16
|
+
quantity: str | None = None
|
|
17
|
+
description: str | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FunctionRequest(BaseModel):
|
|
21
|
+
author_name: str = "unknown"
|
|
22
|
+
author_email: str = "unknown"
|
|
23
|
+
|
|
24
|
+
name: str
|
|
25
|
+
artifact_type: str = "function"
|
|
26
|
+
category: str
|
|
27
|
+
keywords: list[str]
|
|
28
|
+
|
|
29
|
+
homepage_url: str = ""
|
|
30
|
+
documentation_url: str = ""
|
|
31
|
+
source_url: str = ""
|
|
32
|
+
|
|
33
|
+
python_import: str
|
|
34
|
+
dependencies: list[str] | None = None
|
|
35
|
+
|
|
36
|
+
source_code: str
|
|
37
|
+
|
|
38
|
+
docstring: str
|
|
39
|
+
inputs: list[Annotation]
|
|
40
|
+
outputs: list[Annotation]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class FunctionResponse(BaseModel):
|
|
44
|
+
author_name: str
|
|
45
|
+
author_email: str
|
|
46
|
+
|
|
47
|
+
creator_name: str
|
|
48
|
+
creator_email: str
|
|
49
|
+
creation_timestamp: str
|
|
50
|
+
|
|
51
|
+
id: str
|
|
52
|
+
name: str
|
|
53
|
+
artifact_type: str
|
|
54
|
+
category: str
|
|
55
|
+
keywords: list[str]
|
|
56
|
+
|
|
57
|
+
homepage_url: str | None = None
|
|
58
|
+
documentation_url: str | None = None
|
|
59
|
+
source_url: str | None = None
|
|
60
|
+
|
|
61
|
+
python_import: str
|
|
62
|
+
dependencies: list[str] | None = None
|
|
63
|
+
|
|
64
|
+
source_code: str
|
|
65
|
+
|
|
66
|
+
docstring: str
|
|
67
|
+
brief_description: str | None = None
|
|
68
|
+
description: str | None = None
|
|
69
|
+
inputs: list[Annotation]
|
|
70
|
+
outputs: list[Annotation]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ScoredSearchItem(BaseModel):
|
|
74
|
+
score: float
|
|
75
|
+
node: FunctionResponse
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class SearchResults(BaseModel):
|
|
79
|
+
data: list[ScoredSearchItem]
|
|
80
|
+
page: int
|
|
81
|
+
limit: int
|
|
82
|
+
total_items: int
|
|
83
|
+
total_pages: int
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class ScoredSearchResponse(BaseModel):
|
|
87
|
+
results: SearchResults
|
|
88
|
+
aggregations: dict[str, dict[str, int]] | None = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class FunctionMetadata(FunctionResponse):
|
|
92
|
+
embedding: list[float] | None = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class Filter(BaseModel):
|
|
96
|
+
category: str | None = None
|
|
97
|
+
artifact_type: list[ArtifactType] | None = None
|
|
98
|
+
author: list[str] | None = None
|
|
99
|
+
keywords: list[str] | None = None
|
|
100
|
+
datatypes: list[str] | None = None
|
|
101
|
+
units: list[str] | None = None
|
|
102
|
+
quantities: list[str] | None = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class FilterOptions(BaseModel):
|
|
106
|
+
category: dict[str, list[str]]
|
|
107
|
+
artifact_type: list[ArtifactType]
|
|
108
|
+
author: list[str]
|
|
109
|
+
keywords: list[str]
|
|
110
|
+
datatypes: list[str]
|
|
111
|
+
units: list[str]
|
|
112
|
+
quantities: list[str]
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import importlib
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
import inspect
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from http import HTTPStatus
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from types import ModuleType
|
|
10
|
+
from typing import Any, Literal
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from sim_atlas_toolkit.parsers.metadata import Metadata
|
|
15
|
+
|
|
16
|
+
from .models import (
|
|
17
|
+
FunctionRequest,
|
|
18
|
+
)
|
|
19
|
+
from .parser import get_metadata
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger("SimAtlas")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class NodeStore:
|
|
25
|
+
def __init__(self, api_url: str, api_key: str | None = None) -> None:
|
|
26
|
+
self.api_url = api_url
|
|
27
|
+
self.api_key = api_key
|
|
28
|
+
|
|
29
|
+
def upload_module( # noqa: PLR0912
|
|
30
|
+
self,
|
|
31
|
+
module: str | ModuleType,
|
|
32
|
+
update_existing: bool = False,
|
|
33
|
+
parsers: list[Callable[[Any], list[Metadata]]] | None = None,
|
|
34
|
+
recursive: Literal["no", "import", "filesystem"] = "no",
|
|
35
|
+
**kwargs: dict[str, Any],
|
|
36
|
+
) -> None:
|
|
37
|
+
try:
|
|
38
|
+
if isinstance(module, str):
|
|
39
|
+
module = importlib.import_module(module)
|
|
40
|
+
except Exception as e:
|
|
41
|
+
logger.error(f"Failed to import module {module}: {e}")
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
if recursive == "filesystem":
|
|
45
|
+
module_path = Path(module.__path__[0])
|
|
46
|
+
submodule_paths = module_path.glob("**/*.py")
|
|
47
|
+
submodule_paths = (
|
|
48
|
+
path for path in submodule_paths if not path.name.startswith("_")
|
|
49
|
+
)
|
|
50
|
+
submodule_paths = (
|
|
51
|
+
".".join(
|
|
52
|
+
[module.__name__]
|
|
53
|
+
+ str(path.relative_to(module_path).with_suffix("")).split("/")
|
|
54
|
+
)
|
|
55
|
+
for path in submodule_paths
|
|
56
|
+
)
|
|
57
|
+
for submodule_path in submodule_paths:
|
|
58
|
+
self.upload_module(
|
|
59
|
+
submodule_path,
|
|
60
|
+
update_existing=update_existing,
|
|
61
|
+
parsers=parsers,
|
|
62
|
+
recursive="no",
|
|
63
|
+
**kwargs,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
if hasattr(module, "__all__"):
|
|
67
|
+
items = ((k, getattr(module, k)) for k in module.__all__)
|
|
68
|
+
else:
|
|
69
|
+
items = module.__dict__.items()
|
|
70
|
+
|
|
71
|
+
num_uploads = 0
|
|
72
|
+
successful_uploads = 0
|
|
73
|
+
for k, v in items:
|
|
74
|
+
if inspect.ismodule(v):
|
|
75
|
+
if recursive == "import" and v.__name__.startswith(module.__name__):
|
|
76
|
+
self.upload_module(
|
|
77
|
+
v,
|
|
78
|
+
update_existing=update_existing,
|
|
79
|
+
parsers=parsers,
|
|
80
|
+
recursive=recursive,
|
|
81
|
+
**kwargs,
|
|
82
|
+
)
|
|
83
|
+
continue
|
|
84
|
+
|
|
85
|
+
if not hasattr(v, "__module__"):
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
if not v.__module__.startswith(module.__name__):
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
if k.startswith("_"):
|
|
92
|
+
continue
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
responses = self.upload(
|
|
96
|
+
v, update_existing=update_existing, parsers=parsers, **kwargs
|
|
97
|
+
)
|
|
98
|
+
num_uploads += len(responses)
|
|
99
|
+
for response in responses:
|
|
100
|
+
if response.status_code == HTTPStatus.CREATED:
|
|
101
|
+
successful_uploads += 1
|
|
102
|
+
else:
|
|
103
|
+
logger.debug(
|
|
104
|
+
f"Failed to upload {k}: {response.status_code} {response.text}"
|
|
105
|
+
)
|
|
106
|
+
except Exception as e:
|
|
107
|
+
logger.debug(f"Failed to upload {k}: {v}\n{e}")
|
|
108
|
+
continue
|
|
109
|
+
|
|
110
|
+
logger.info(f"{successful_uploads}/{num_uploads} uploaded from {module}")
|
|
111
|
+
|
|
112
|
+
def upload( # noqa: PLR0912
|
|
113
|
+
self,
|
|
114
|
+
obj: Any,
|
|
115
|
+
update_existing: bool = False,
|
|
116
|
+
parsers: list[Callable[[Any], list[Metadata]]] | None = None,
|
|
117
|
+
**kwargs: dict[str, Any],
|
|
118
|
+
) -> list[requests.Response]:
|
|
119
|
+
headers: dict[str, str] = {}
|
|
120
|
+
if self.api_key:
|
|
121
|
+
headers["x-api-key"] = self.api_key
|
|
122
|
+
|
|
123
|
+
if isinstance(obj, FunctionRequest):
|
|
124
|
+
response = requests.post(
|
|
125
|
+
f"{self.api_url}/artifacts",
|
|
126
|
+
json=obj.model_dump(),
|
|
127
|
+
headers=headers,
|
|
128
|
+
)
|
|
129
|
+
return [response]
|
|
130
|
+
|
|
131
|
+
if inspect.ismodule(obj):
|
|
132
|
+
raise ValueError(
|
|
133
|
+
"Will not automatically upload modules. Use upload_module instead."
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
general_metadata: dict[str, Any] = {}
|
|
137
|
+
|
|
138
|
+
with contextlib.suppress(Exception):
|
|
139
|
+
if dependencies := importlib.metadata.requires(
|
|
140
|
+
obj.__module__.partition(".")[0]
|
|
141
|
+
):
|
|
142
|
+
general_metadata["dependencies"] = dependencies
|
|
143
|
+
|
|
144
|
+
try:
|
|
145
|
+
package_metadata = importlib.metadata.metadata(
|
|
146
|
+
obj.__module__.partition(".")[0]
|
|
147
|
+
).json
|
|
148
|
+
except Exception as _:
|
|
149
|
+
package_metadata = {}
|
|
150
|
+
|
|
151
|
+
if author := package_metadata.get("author"):
|
|
152
|
+
general_metadata["author_name"] = author
|
|
153
|
+
if email := package_metadata.get("author_email"):
|
|
154
|
+
general_metadata["author_email"] = email
|
|
155
|
+
|
|
156
|
+
with contextlib.suppress(Exception):
|
|
157
|
+
project_url = package_metadata.get("project_url")
|
|
158
|
+
if project_url is not None:
|
|
159
|
+
for item in project_url:
|
|
160
|
+
key, url = item.split(", ")
|
|
161
|
+
if key.lower() in ["homepage"]:
|
|
162
|
+
general_metadata["homepage_url"] = url
|
|
163
|
+
continue
|
|
164
|
+
if key.lower() in ["documentation"]:
|
|
165
|
+
general_metadata["documentation_url"] = url
|
|
166
|
+
continue
|
|
167
|
+
if key.lower() in ["source", "code", "repository", "github"]:
|
|
168
|
+
general_metadata["source_url"] = url
|
|
169
|
+
continue
|
|
170
|
+
|
|
171
|
+
metadata_list = get_metadata(obj, parsers)
|
|
172
|
+
responses: list[requests.Response] = []
|
|
173
|
+
for metadata in metadata_list:
|
|
174
|
+
metadata_dict = metadata.model_dump()
|
|
175
|
+
metadata_dict.update(general_metadata)
|
|
176
|
+
metadata_dict.update(kwargs)
|
|
177
|
+
|
|
178
|
+
request_data = FunctionRequest.model_validate(metadata_dict)
|
|
179
|
+
|
|
180
|
+
responses.append(
|
|
181
|
+
requests.post(
|
|
182
|
+
f"{self.api_url}/artifacts",
|
|
183
|
+
json=request_data.model_dump(),
|
|
184
|
+
headers=headers,
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
return responses
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .parsers import (
|
|
5
|
+
dataclass_node,
|
|
6
|
+
pyiron_core,
|
|
7
|
+
pyiron_workflow,
|
|
8
|
+
python_function,
|
|
9
|
+
python_workflow_definition,
|
|
10
|
+
)
|
|
11
|
+
from .parsers.metadata import Metadata
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_metadata(
|
|
15
|
+
obj: Any, parsers: list[Callable[[Any], list[Metadata]]] | None
|
|
16
|
+
) -> list[Metadata]:
|
|
17
|
+
if parsers is None:
|
|
18
|
+
parsers = [
|
|
19
|
+
dataclass_node.parse,
|
|
20
|
+
pyiron_workflow.parse,
|
|
21
|
+
pyiron_core.parse,
|
|
22
|
+
python_workflow_definition.parse,
|
|
23
|
+
python_function.parse,
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
for parser in parsers:
|
|
27
|
+
if metadata := parser(obj):
|
|
28
|
+
return metadata
|
|
29
|
+
|
|
30
|
+
raise ValueError(f"No parser available for the given object: {type(obj)}")
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
import inspect
|
|
3
|
+
import textwrap
|
|
4
|
+
from typing import Any, get_type_hints
|
|
5
|
+
|
|
6
|
+
from ..models import Annotation, ArtifactType
|
|
7
|
+
from .metadata import Metadata, parse_annotation
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _field_annotations(cls: type) -> list[Annotation]:
|
|
11
|
+
try:
|
|
12
|
+
hints = get_type_hints(cls, include_extras=True)
|
|
13
|
+
except Exception:
|
|
14
|
+
hints = {}
|
|
15
|
+
|
|
16
|
+
result: list[Annotation] = []
|
|
17
|
+
for f in dataclasses.fields(cls): # type: ignore[arg-type]
|
|
18
|
+
if not f.init:
|
|
19
|
+
continue
|
|
20
|
+
tp = hints.get(f.name)
|
|
21
|
+
has_default = (
|
|
22
|
+
f.default is not dataclasses.MISSING
|
|
23
|
+
or f.default_factory is not dataclasses.MISSING # type: ignore[misc]
|
|
24
|
+
)
|
|
25
|
+
ann = parse_annotation(tp) if tp is not None else Annotation()
|
|
26
|
+
result.append(
|
|
27
|
+
Annotation(
|
|
28
|
+
label=ann.label if ann.label is not None else f.name,
|
|
29
|
+
datatype=ann.datatype,
|
|
30
|
+
unit=ann.unit,
|
|
31
|
+
quantity=ann.quantity,
|
|
32
|
+
has_default_value=has_default,
|
|
33
|
+
)
|
|
34
|
+
)
|
|
35
|
+
return result
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def parse(obj: Any) -> list[Metadata]:
|
|
39
|
+
if not (dataclasses.is_dataclass(obj) and isinstance(obj, type)):
|
|
40
|
+
return []
|
|
41
|
+
|
|
42
|
+
module: str = obj.__module__
|
|
43
|
+
qualname: str = obj.__qualname__
|
|
44
|
+
python_import = f"{module}.{qualname}"
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
raw_source = textwrap.dedent(inspect.getsource(obj).replace("\\r\\n", ""))
|
|
48
|
+
except OSError:
|
|
49
|
+
return []
|
|
50
|
+
|
|
51
|
+
pack_note = (
|
|
52
|
+
f"# NOTE: This is a virtual PACK node for {qualname}.\n"
|
|
53
|
+
"# It takes the dataclass fields as individual inputs and constructs the dataclass instance.\n"
|
|
54
|
+
"# The source code below is shown for reference only.\n"
|
|
55
|
+
)
|
|
56
|
+
unpack_note = (
|
|
57
|
+
f"# NOTE: This is a virtual UNPACK node for {qualname}.\n"
|
|
58
|
+
"# It takes a dataclass instance as input and exposes its fields as individual outputs.\n"
|
|
59
|
+
"# The source code below is shown for reference only.\n"
|
|
60
|
+
)
|
|
61
|
+
pack_source = pack_note + raw_source
|
|
62
|
+
unpack_source = unpack_note + raw_source
|
|
63
|
+
|
|
64
|
+
category = module.replace(".", ">")
|
|
65
|
+
raw_doc = inspect.getdoc(obj) or ""
|
|
66
|
+
|
|
67
|
+
field_annotations = _field_annotations(obj)
|
|
68
|
+
dataclass_annotation = Annotation(label=qualname, datatype=python_import)
|
|
69
|
+
|
|
70
|
+
return [
|
|
71
|
+
Metadata(
|
|
72
|
+
name=f"[PACK] {python_import}",
|
|
73
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
74
|
+
python_import=python_import,
|
|
75
|
+
category=category,
|
|
76
|
+
source_code=pack_source,
|
|
77
|
+
docstring=f"[PACK] {qualname}: {raw_doc}",
|
|
78
|
+
keywords=["pack"],
|
|
79
|
+
inputs=field_annotations,
|
|
80
|
+
outputs=[dataclass_annotation],
|
|
81
|
+
),
|
|
82
|
+
Metadata(
|
|
83
|
+
name=f"[UNPACK] {python_import}",
|
|
84
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
85
|
+
python_import=python_import,
|
|
86
|
+
category=category,
|
|
87
|
+
source_code=unpack_source,
|
|
88
|
+
docstring=f"[UNPACK] {qualname}: {raw_doc}",
|
|
89
|
+
keywords=["unpack"],
|
|
90
|
+
inputs=[dataclass_annotation],
|
|
91
|
+
outputs=field_annotations,
|
|
92
|
+
),
|
|
93
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import types
|
|
3
|
+
from typing import Annotated, Any, Union, get_args, get_origin
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
from ..models import Annotation, ArtifactType
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Metadata(BaseModel):
|
|
11
|
+
name: str | None = None
|
|
12
|
+
artifact_type: ArtifactType
|
|
13
|
+
category: str
|
|
14
|
+
|
|
15
|
+
python_import: str | None
|
|
16
|
+
source_code: str
|
|
17
|
+
|
|
18
|
+
keywords: list[str]
|
|
19
|
+
|
|
20
|
+
docstring: str
|
|
21
|
+
inputs: list[Annotation]
|
|
22
|
+
outputs: list[Annotation]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def type_to_str(tp: Any) -> str:
|
|
26
|
+
"""Recursively convert a Python type annotation to a canonical string.
|
|
27
|
+
|
|
28
|
+
Examples:
|
|
29
|
+
int -> "int"
|
|
30
|
+
type(None) -> "None"
|
|
31
|
+
list[int] -> "list[int]"
|
|
32
|
+
dict[str, float] -> "dict[str, float]"
|
|
33
|
+
int | float -> "int | float"
|
|
34
|
+
Optional[int] -> "int | None"
|
|
35
|
+
"""
|
|
36
|
+
if tp is type(None):
|
|
37
|
+
return "None"
|
|
38
|
+
|
|
39
|
+
origin = get_origin(tp)
|
|
40
|
+
args = get_args(tp)
|
|
41
|
+
|
|
42
|
+
# Union: PEP 604 (int | float) or typing.Union / Optional
|
|
43
|
+
if origin is Union or isinstance(tp, types.UnionType):
|
|
44
|
+
return " | ".join(type_to_str(a) for a in args)
|
|
45
|
+
|
|
46
|
+
# Generic alias: list[int], dict[str, float], etc.
|
|
47
|
+
if origin is not None and args:
|
|
48
|
+
name = getattr(origin, "__name__", str(origin))
|
|
49
|
+
return f"{name}[{', '.join(type_to_str(a) for a in args)}]"
|
|
50
|
+
|
|
51
|
+
# Simple type
|
|
52
|
+
return getattr(tp, "__name__", str(tp))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_annotation(annotation: Any) -> Annotation:
|
|
56
|
+
"""Parse a type annotation to extract datatype, unit, and quantity.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
annotation: The type annotation to parse.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
Annotation: The parsed annotation details.
|
|
63
|
+
"""
|
|
64
|
+
if annotation is None:
|
|
65
|
+
return Annotation()
|
|
66
|
+
|
|
67
|
+
if annotation == inspect.Parameter.empty:
|
|
68
|
+
return Annotation()
|
|
69
|
+
|
|
70
|
+
origin = get_origin(annotation)
|
|
71
|
+
if origin is not Annotated:
|
|
72
|
+
return Annotation(datatype=type_to_str(annotation))
|
|
73
|
+
|
|
74
|
+
args = get_args(annotation)
|
|
75
|
+
result = Annotation(datatype=type_to_str(args[0]))
|
|
76
|
+
for arg in args[1:]:
|
|
77
|
+
if isinstance(arg, dict):
|
|
78
|
+
result.label = arg.get("label", result.label) # type: ignore
|
|
79
|
+
result.unit = arg.get("unit", result.unit) # type: ignore
|
|
80
|
+
result.quantity = arg.get("quantity", result.quantity) # type: ignore
|
|
81
|
+
result.description = arg.get("description", result.description) # type: ignore
|
|
82
|
+
return result
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from ..models import Annotation, ArtifactType
|
|
5
|
+
from .metadata import Metadata
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def parse(obj: Any) -> list[Metadata]:
|
|
9
|
+
if not hasattr(obj, "__globals__"):
|
|
10
|
+
return []
|
|
11
|
+
if obj.__globals__["__name__"] != "pyiron_core.pyiron_workflow.simple_workflow":
|
|
12
|
+
return []
|
|
13
|
+
|
|
14
|
+
source_code = inspect.getsource(obj)
|
|
15
|
+
|
|
16
|
+
instance = obj()
|
|
17
|
+
|
|
18
|
+
inputs = [Annotation(label=k, datatype=v.type) for k, v in instance.inputs.items()]
|
|
19
|
+
outputs = [
|
|
20
|
+
Annotation(label=k, datatype=v.type) for k, v in instance.outputs.items()
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
return [
|
|
24
|
+
Metadata(
|
|
25
|
+
name=f"{instance._func.__module__}.{instance._func.__qualname__}",
|
|
26
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
27
|
+
python_import=f"{instance._func.__module__}.{instance._func.__qualname__}",
|
|
28
|
+
category=f"{instance._func.__module__}".replace(".", ">"),
|
|
29
|
+
source_code=source_code,
|
|
30
|
+
docstring=instance._func.__doc__ or "",
|
|
31
|
+
keywords=["pyiron_core_node"],
|
|
32
|
+
inputs=inputs,
|
|
33
|
+
outputs=outputs,
|
|
34
|
+
)
|
|
35
|
+
]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# pyright: basic
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from ..models import Annotation, ArtifactType
|
|
7
|
+
from .metadata import Metadata, parse_annotation
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def parse(node: Any) -> list[Metadata]:
|
|
11
|
+
if not isinstance(node, type):
|
|
12
|
+
return []
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from pyiron_workflow.api import NOT_DATA # noqa: PLC0415
|
|
16
|
+
from pyiron_workflow.nodes.function import Function # noqa: PLC0415
|
|
17
|
+
except ImportError:
|
|
18
|
+
return []
|
|
19
|
+
|
|
20
|
+
if not issubclass(node, Function):
|
|
21
|
+
return []
|
|
22
|
+
|
|
23
|
+
source_code = inspect.getsource(node.node_function)
|
|
24
|
+
|
|
25
|
+
inputs: list[Annotation] = []
|
|
26
|
+
outputs: list[Annotation] = []
|
|
27
|
+
for k, v in node.preview_inputs().items():
|
|
28
|
+
ann = parse_annotation(v[0])
|
|
29
|
+
ann.label = k
|
|
30
|
+
ann.has_default_value = v[1] != NOT_DATA
|
|
31
|
+
inputs.append(ann)
|
|
32
|
+
for k, v in node.preview_outputs().items():
|
|
33
|
+
ann = parse_annotation(v)
|
|
34
|
+
ann.label = k
|
|
35
|
+
outputs.append(ann)
|
|
36
|
+
|
|
37
|
+
return [
|
|
38
|
+
Metadata(
|
|
39
|
+
name=f"{node.node_function.__module__}.{node.node_function.__qualname__}",
|
|
40
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
41
|
+
python_import=f"{node.node_function.__module__}.{node.node_function.__qualname__}",
|
|
42
|
+
category=f"{node.node_function.__module__}".replace(".", ">"),
|
|
43
|
+
source_code=source_code,
|
|
44
|
+
docstring=node.node_function.__doc__ or "",
|
|
45
|
+
keywords=["pyiron_workflow_function"],
|
|
46
|
+
inputs=inputs,
|
|
47
|
+
outputs=outputs,
|
|
48
|
+
)
|
|
49
|
+
]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import textwrap
|
|
3
|
+
from typing import Annotated, Any, get_args, get_origin
|
|
4
|
+
|
|
5
|
+
from ..models import Annotation, ArtifactType
|
|
6
|
+
from .metadata import Metadata, parse_annotation
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _parse_arguments(sig: inspect.Signature) -> list[Annotation]:
|
|
10
|
+
arguments: list[Annotation] = []
|
|
11
|
+
for param_name, param in sig.parameters.items():
|
|
12
|
+
ann = parse_annotation(param.annotation)
|
|
13
|
+
ann.label = param_name
|
|
14
|
+
arguments.append(ann)
|
|
15
|
+
return arguments
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parse_and_unpack_annotation(annotation: Any) -> list[Annotation]:
|
|
19
|
+
origin = get_origin(annotation)
|
|
20
|
+
args = get_args(annotation)
|
|
21
|
+
|
|
22
|
+
# unpack one level of Annotated
|
|
23
|
+
if origin is Annotated:
|
|
24
|
+
origin = get_origin(args[0])
|
|
25
|
+
args = get_args(args[0])
|
|
26
|
+
|
|
27
|
+
if origin is tuple:
|
|
28
|
+
annotations: list[Annotation] = []
|
|
29
|
+
for i, arg in enumerate(args):
|
|
30
|
+
ann = parse_annotation(arg)
|
|
31
|
+
ann.label = ann.label if ann.label is not None else str(i)
|
|
32
|
+
annotations.append(ann)
|
|
33
|
+
return annotations
|
|
34
|
+
|
|
35
|
+
ann = parse_annotation(annotation)
|
|
36
|
+
if not ann.label:
|
|
37
|
+
ann.label = "return"
|
|
38
|
+
return [ann]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse(obj: Any) -> list[Metadata]:
|
|
42
|
+
if not (inspect.isfunction(obj) or inspect.isbuiltin(obj)):
|
|
43
|
+
return []
|
|
44
|
+
|
|
45
|
+
if inspect.isbuiltin(obj):
|
|
46
|
+
source_code = f"{obj.__name__}{inspect.signature(obj)}"
|
|
47
|
+
else:
|
|
48
|
+
source_code = inspect.getsource(obj)
|
|
49
|
+
source_code = textwrap.dedent(source_code.replace("\\r\\n", ""))
|
|
50
|
+
|
|
51
|
+
sig = inspect.signature(obj)
|
|
52
|
+
inputs = _parse_arguments(sig)
|
|
53
|
+
|
|
54
|
+
return_annotation = sig.return_annotation
|
|
55
|
+
outputs = _parse_and_unpack_annotation(return_annotation)
|
|
56
|
+
|
|
57
|
+
return [
|
|
58
|
+
Metadata(
|
|
59
|
+
name=f"{obj.__module__}.{obj.__qualname__}",
|
|
60
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
61
|
+
python_import=f"{obj.__module__}.{obj.__qualname__}",
|
|
62
|
+
category=f"{obj.__module__}".replace(".", ">"),
|
|
63
|
+
source_code=source_code,
|
|
64
|
+
docstring=inspect.getdoc(obj) or "",
|
|
65
|
+
keywords=[],
|
|
66
|
+
inputs=inputs,
|
|
67
|
+
outputs=outputs,
|
|
68
|
+
)
|
|
69
|
+
]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# pyright: basic
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from ..models import Annotation, ArtifactType
|
|
6
|
+
from .metadata import Metadata
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def parse(obj: Any) -> list[Metadata]:
|
|
10
|
+
try:
|
|
11
|
+
from python_workflow_definition.models import ( # noqa: PLC0415
|
|
12
|
+
PythonWorkflowDefinitionInputNode,
|
|
13
|
+
PythonWorkflowDefinitionOutputNode,
|
|
14
|
+
PythonWorkflowDefinitionWorkflow,
|
|
15
|
+
)
|
|
16
|
+
except ImportError:
|
|
17
|
+
return []
|
|
18
|
+
|
|
19
|
+
if not isinstance(obj, PythonWorkflowDefinitionWorkflow):
|
|
20
|
+
return []
|
|
21
|
+
|
|
22
|
+
source_code: str = obj.model_dump_json(indent=2)
|
|
23
|
+
|
|
24
|
+
inputs: list[Annotation] = []
|
|
25
|
+
outputs: list[Annotation] = []
|
|
26
|
+
for node in obj.nodes:
|
|
27
|
+
if isinstance(node, PythonWorkflowDefinitionInputNode):
|
|
28
|
+
ann = Annotation(label=node.name)
|
|
29
|
+
inputs.append(ann)
|
|
30
|
+
if isinstance(node, PythonWorkflowDefinitionOutputNode):
|
|
31
|
+
ann = Annotation(label=node.name)
|
|
32
|
+
outputs.append(ann)
|
|
33
|
+
|
|
34
|
+
return [
|
|
35
|
+
Metadata(
|
|
36
|
+
artifact_type=ArtifactType.FUNCTION,
|
|
37
|
+
python_import="",
|
|
38
|
+
category="workflow",
|
|
39
|
+
source_code=source_code,
|
|
40
|
+
docstring="",
|
|
41
|
+
keywords=[],
|
|
42
|
+
inputs=inputs,
|
|
43
|
+
outputs=outputs,
|
|
44
|
+
)
|
|
45
|
+
]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sim-atlas-toolkit
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Tools for parsing and storing workflow nodes with support for pyiron and Python functions
|
|
5
|
+
Keywords: pyiron
|
|
6
|
+
Author: Sebastian Eibl
|
|
7
|
+
Author-email: Sebastian Eibl <sebastian.eibl@mpcdf.mpg.de>
|
|
8
|
+
License-Expression: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Dist: pydantic>=2
|
|
22
|
+
Requires-Dist: requests>=2
|
|
23
|
+
Maintainer: Sebastian Eibl
|
|
24
|
+
Maintainer-email: Sebastian Eibl <sebastian.eibl@mpcdf.mpg.de>
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
sim_atlas_toolkit/__init__.py,sha256=ToaI3TG03Zw_BgjCBBpkZleLXK0TvskMJjA_apRxuCI,59
|
|
2
|
+
sim_atlas_toolkit/cli.py,sha256=PwCDG3tQumbb194xAhYFJGVCn5uKg4AuH0jrShWH7vs,2776
|
|
3
|
+
sim_atlas_toolkit/models.py,sha256=_07d8lD8Sj0C-xfUcDMXNPprDiJ26nkJqZEodUsmvck,2366
|
|
4
|
+
sim_atlas_toolkit/node_store.py,sha256=BP31mjx2SrOSigMK2Cp9X_H5aZ8Zj4EYiu9a5MJ03LQ,6389
|
|
5
|
+
sim_atlas_toolkit/parser.py,sha256=feCO4rF3IhAzgVVxqpt8LX-SL3pZV1kOcP1KSJ2vuqY,752
|
|
6
|
+
sim_atlas_toolkit/parsers/dataclass_node.py,sha256=q1kLKNt1tyfEh_rzesd9J7OYRQvyyHBsb1EDUkTphmQ,3084
|
|
7
|
+
sim_atlas_toolkit/parsers/metadata.py,sha256=1qHPuzzkqxlh04OPMxhoGNAiNelB1HbCbPvmmpmOaJc,2367
|
|
8
|
+
sim_atlas_toolkit/parsers/pyiron_core.py,sha256=Jy0FPupVGUNV0l74tJsig83gF-v_Ge21li5it-sOxKY,1098
|
|
9
|
+
sim_atlas_toolkit/parsers/pyiron_workflow.py,sha256=amnia--L9j_CrH_VcUaDoEJ-jy29iLgwCnVeD1HK0mE,1496
|
|
10
|
+
sim_atlas_toolkit/parsers/python_function.py,sha256=muD9xHj5NcqmfKUSoDcD5NDJm1EvVmT7mGuTcfKcwb8,2105
|
|
11
|
+
sim_atlas_toolkit/parsers/python_workflow_definition.py,sha256=CKVf5EPOLTGsBFy91kO2m3VVICQa5l7qPG8O-O1qcWg,1255
|
|
12
|
+
sim_atlas_toolkit-0.0.1.dist-info/licenses/LICENSE,sha256=3XlzskqO4cZlDwSVZwRKaGo47u4TLj3TSlvn5Uj7yEk,1501
|
|
13
|
+
sim_atlas_toolkit-0.0.1.dist-info/WHEEL,sha256=iHtWm8nRfs0VRdCYVXocAWFW8ppjHL-uTJkAdZJKOBM,80
|
|
14
|
+
sim_atlas_toolkit-0.0.1.dist-info/entry_points.txt,sha256=YFMaD3t7ZwazloKxw7756Gh536jj-bMAZ6k2zyKaSmk,65
|
|
15
|
+
sim_atlas_toolkit-0.0.1.dist-info/METADATA,sha256=s7xbfdd3rYK-UzU01hnk9JnxkeyqjayAOwE3X-6M1hU,1069
|
|
16
|
+
sim_atlas_toolkit-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Sebastian Eibl
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|