tinyfan 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinyfan-0.0.2 → tinyfan-0.1.0}/PKG-INFO +1 -1
- tinyfan-0.1.0/examples/configmap.yaml +7 -0
- tinyfan-0.1.0/examples/configs.py +31 -0
- tinyfan-0.1.0/examples/packaged/Dockerfile +5 -0
- tinyfan-0.1.0/examples/packaged/pyproject.toml +13 -0
- tinyfan-0.1.0/examples/packaged/src/tinyfan_examples_packaged/asset.py +23 -0
- tinyfan-0.1.0/examples/resource.py +41 -0
- tinyfan-0.1.0/examples/secret.yaml +7 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/pyproject.toml +1 -1
- tinyfan-0.1.0/src/tinyfan/__init__.py +19 -0
- tinyfan-0.1.0/src/tinyfan/cli.py +29 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/codegen.py +82 -5
- tinyfan-0.1.0/src/tinyfan/config.py +97 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/flow.py +25 -19
- tinyfan-0.1.0/src/tinyfan/resources/base.py +12 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/base.py +3 -2
- tinyfan-0.1.0/tests/codegen_samples/directory/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/test_codegen.py +46 -15
- tinyfan-0.1.0/tests/utils/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/uv.lock +1 -1
- tinyfan-0.0.2/examples/realworld/pyproject.toml +0 -7
- tinyfan-0.0.2/src/tinyfan/__init__.py +0 -5
- tinyfan-0.0.2/src/tinyfan/cli.py +0 -24
- {tinyfan-0.0.2 → tinyfan-0.1.0}/.github/workflows/publish.yml +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/.gitignore +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/.python-version +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/README.md +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/examples/getting-started.py +0 -0
- {tinyfan-0.0.2/examples/realworld → tinyfan-0.1.0/examples/packaged}/.python-version +0 -0
- {tinyfan-0.0.2/examples/realworld → tinyfan-0.1.0/examples/packaged}/README.md +0 -0
- {tinyfan-0.0.2/src/tinyfan/stores → tinyfan-0.1.0/examples/packaged/src/tinyfan_examples_packaged}/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/argo_typing.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/flowrundata.py +0 -0
- {tinyfan-0.0.2/src/tinyfan/utils → tinyfan-0.1.0/src/tinyfan/resources}/__init__.py +0 -0
- {tinyfan-0.0.2/tests → tinyfan-0.1.0/src/tinyfan/stores}/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/bigquery.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/gcs_pickle.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/naive.py +0 -0
- {tinyfan-0.0.2/tests/codegen_samples/directory → tinyfan-0.1.0/src/tinyfan/utils}/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/embed.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/exjson.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/merge.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/typing.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/yaml.py +0 -0
- {tinyfan-0.0.2/tests/utils → tinyfan-0.1.0/tests}/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/directory/asset.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/directory/asset2.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/singlefile.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/conftest.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/helpers.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/openapi2jsonschema.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/test_decorators.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/directory/__init__.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/directory/dep.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/singlefile.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/test_embed.py +0 -0
- {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/test_exjson.py +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import tinyfan
|
|
2
|
+
from tinyfan import ConfigMapRef, ConfigMapKeyRef, SecretRef, SecretKeyRef, cli_arg
|
|
3
|
+
|
|
4
|
+
flow = tinyfan.Flow(
|
|
5
|
+
name="configs",
|
|
6
|
+
configs={
|
|
7
|
+
"constant": "const",
|
|
8
|
+
"configmap": ConfigMapRef("tinyfan-config-example"),
|
|
9
|
+
"configmapkey": ConfigMapKeyRef("tinyfan-config-example", "key"),
|
|
10
|
+
"secret": SecretRef("tinyfan-config-example"),
|
|
11
|
+
"secretkey": SecretKeyRef("tinyfan-config-example", "secretkey"),
|
|
12
|
+
"cliarg": cli_arg("example"),
|
|
13
|
+
},
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@tinyfan.asset(
|
|
18
|
+
schedule="*/1 * * * *",
|
|
19
|
+
flow=flow,
|
|
20
|
+
)
|
|
21
|
+
def trace_configs(
|
|
22
|
+
constant: str,
|
|
23
|
+
cliarg: str,
|
|
24
|
+
configmap: ConfigMapRef,
|
|
25
|
+
configmapkey: ConfigMapKeyRef,
|
|
26
|
+
secret: SecretRef,
|
|
27
|
+
secretkey: SecretKeyRef,
|
|
28
|
+
):
|
|
29
|
+
print(
|
|
30
|
+
f'{constant} - {cliarg} - {configmap.get("otherkey")} - {configmapkey.get_value()} - {secret.get("othersecretkey")} - {secretkey.get_value()}'
|
|
31
|
+
)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "tinyfan_examples_packaged"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "tinyfan pacakged example"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
dependencies = ["tinyfan"]
|
|
8
|
+
|
|
9
|
+
[build-system]
|
|
10
|
+
requires = ["hatchling"]
|
|
11
|
+
build-backend = "hatchling.build"
|
|
12
|
+
|
|
13
|
+
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import tinyfan
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@tinyfan.asset(
|
|
6
|
+
schedule="*/1 * * * *", container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"}
|
|
7
|
+
)
|
|
8
|
+
def target() -> str:
|
|
9
|
+
return "world"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@tinyfan.asset(container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"})
|
|
13
|
+
def greeting(target: str):
|
|
14
|
+
print("hello " + target)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@tinyfan.asset(
|
|
18
|
+
schedule="*/1 * * * *", container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"}
|
|
19
|
+
)
|
|
20
|
+
def check_variables(ds: str, ts: str, data_interval_start: datetime, data_interval_end: datetime) -> None:
|
|
21
|
+
print(f"ds: {ds}, ts: {ts}")
|
|
22
|
+
print(f"data_interval_start: {data_interval_start}")
|
|
23
|
+
print(f"data_interval_end: {data_interval_end}")
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import tinyfan
|
|
2
|
+
from tinyfan import ConfigMapKeyRef
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Res(tinyfan.ResourceBase):
|
|
6
|
+
key: ConfigMapKeyRef
|
|
7
|
+
|
|
8
|
+
def __init__(self, key: ConfigMapKeyRef):
|
|
9
|
+
self.key = key
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ResContainer(tinyfan.ResourceBase):
|
|
13
|
+
res: Res
|
|
14
|
+
|
|
15
|
+
def __init__(self, res: Res):
|
|
16
|
+
self.res = res
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
flow = tinyfan.Flow(
|
|
20
|
+
name="configs",
|
|
21
|
+
resources={
|
|
22
|
+
"res": Res(ConfigMapKeyRef("tinyfan-config-example", "key")),
|
|
23
|
+
"res2": ResContainer(Res(ConfigMapKeyRef("tinyfan-config-example", "key"))),
|
|
24
|
+
},
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@tinyfan.asset(
|
|
29
|
+
schedule="*/1 * * * *",
|
|
30
|
+
flow=flow,
|
|
31
|
+
)
|
|
32
|
+
def trace_res(res: Res):
|
|
33
|
+
print(f"{res.key.get_value()}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@tinyfan.asset(
|
|
37
|
+
schedule="*/1 * * * *",
|
|
38
|
+
flow=flow,
|
|
39
|
+
)
|
|
40
|
+
def trace_res2(res2: ResContainer):
|
|
41
|
+
print(f"{res2.res.key.get_value()}")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from .flow import Flow, Asset, asset
|
|
2
|
+
from .stores.base import StoreBase
|
|
3
|
+
from .stores.naive import NaiveStore
|
|
4
|
+
from .config import ConfigMapRef, ConfigMapKeyRef, SecretRef, SecretKeyRef, cli_arg
|
|
5
|
+
from .resources.base import ResourceBase
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Flow",
|
|
9
|
+
"Asset",
|
|
10
|
+
"asset",
|
|
11
|
+
"StoreBase",
|
|
12
|
+
"NaiveStore",
|
|
13
|
+
"ConfigMapRef",
|
|
14
|
+
"ConfigMapKeyRef",
|
|
15
|
+
"SecretRef",
|
|
16
|
+
"SecretKeyRef",
|
|
17
|
+
"cli_arg",
|
|
18
|
+
"ResourceBase",
|
|
19
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
|
|
3
|
+
from typing_extensions import Annotated
|
|
4
|
+
from .codegen import codegen
|
|
5
|
+
from .config import CLI_CONFIG_STATE
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def tinyfan(
|
|
9
|
+
location: str,
|
|
10
|
+
embedded: bool | None = None,
|
|
11
|
+
arg: Annotated[list[str] | None, typer.Option(metavar="KEY=VALUE")] = None,
|
|
12
|
+
):
|
|
13
|
+
"""
|
|
14
|
+
Generate argocd workflow resource as yaml from tinyfan definitions
|
|
15
|
+
"""
|
|
16
|
+
if arg is not None:
|
|
17
|
+
CLI_CONFIG_STATE.cli_args.update({k: v for (k, v) in (val.split("=") for val in arg)})
|
|
18
|
+
if location.endswith(".py"):
|
|
19
|
+
print(codegen(location=location, embedded=embedded or True))
|
|
20
|
+
else:
|
|
21
|
+
print(codegen(location=location, embedded=embedded or False))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main():
|
|
25
|
+
return typer.run(tinyfan)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
if __name__ == "__main__":
|
|
29
|
+
main()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from tinyfan.flow import Flow, Asset,
|
|
1
|
+
from tinyfan.flow import Flow, Asset, FLOW_REGISTER, DEFAULT_IMAGE
|
|
2
2
|
import importlib.util
|
|
3
3
|
from typing import Self
|
|
4
4
|
from .utils.yaml import dump as yaml_dump, dump_all as yaml_dump_all
|
|
@@ -14,6 +14,7 @@ import croniter
|
|
|
14
14
|
from .argo_typing import ScriptTemplate
|
|
15
15
|
from .utils.embed import embed
|
|
16
16
|
from .utils.merge import deepmerge
|
|
17
|
+
from .config import CLI_CONFIG_STATE, Config, ConfigValue, ConfigMapKeyRef, ConfigMapRef, SecretKeyRef, SecretRef
|
|
17
18
|
|
|
18
19
|
VALID_CRON_REGEXP = r"^(?:(?:(?:(?:\d+,)+\d+|(?:\d+(?:\/|-|#)\d+)|\d+L?|\*(?:\/\d+)?|L(?:-\d+)?|\?|[A-Z]{3}(?:-[A-Z]{3})?) ?){5,7})|(@hourly|@daily|@midnight|@weekly|@monthly|@yearly|@annually)$"
|
|
19
20
|
VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeeded|Failed|Errored|Skipped|Omitted|Daemoned)\w+)(?!.*(?:&&|\|\|)\s*(?:&&|\|\|))(?!.*!!)[A-Za-z0-9_&|!().\s]+$"
|
|
@@ -66,6 +67,7 @@ def import_all_submodules(location: str):
|
|
|
66
67
|
|
|
67
68
|
class AssetNode:
|
|
68
69
|
asset: Asset
|
|
70
|
+
refs: list[Config | ConfigValue | str | None]
|
|
69
71
|
children: list[Self]
|
|
70
72
|
parents: list[Self]
|
|
71
73
|
|
|
@@ -130,8 +132,18 @@ class AssetTree:
|
|
|
130
132
|
for node in self.nodes.values():
|
|
131
133
|
sigs = inspect.signature(node.asset.func)
|
|
132
134
|
func_param_names = list(sigs.parameters.keys())
|
|
133
|
-
|
|
135
|
+
known_names = (
|
|
136
|
+
set(self.flow.assets.keys())
|
|
137
|
+
| set((self.flow.resources or {}).keys())
|
|
138
|
+
| set((self.flow.configs or {}).keys())
|
|
139
|
+
)
|
|
140
|
+
unknown_names = [n for n in func_param_names if n not in known_names]
|
|
141
|
+
if len(unknown_names) > 0:
|
|
142
|
+
raise Exception(
|
|
143
|
+
f"unknown asset args: {', '.join('`' + n + '`' for n in unknown_names)} from asset `{node.asset.name}`"
|
|
144
|
+
)
|
|
134
145
|
|
|
146
|
+
params_ids = set(name for name in func_param_names if name in self.flow.assets)
|
|
135
147
|
if node.asset.depends is not None:
|
|
136
148
|
depends_ids = set(re.findall(EXTRACT_DEPENDS_REGEXP, node.asset.depends))
|
|
137
149
|
node.parents = [self.nodes[n] for n in depends_ids.union(params_ids)]
|
|
@@ -146,6 +158,13 @@ class AssetTree:
|
|
|
146
158
|
p.children.append(node)
|
|
147
159
|
node.asset.depends = " && ".join(params_ids).replace("_", "-")
|
|
148
160
|
|
|
161
|
+
flow_resources = self.flow.resources or {}
|
|
162
|
+
flow_refs = self.flow.configs or {}
|
|
163
|
+
param_resources = [flow_resources[n] for n in func_param_names if n in flow_resources]
|
|
164
|
+
resource_refs = [ref for res in param_resources for ref in res.get_refs()]
|
|
165
|
+
param_refs = [flow_refs[n] for n in func_param_names if n in flow_refs]
|
|
166
|
+
node.refs = list(set(resource_refs + param_refs))
|
|
167
|
+
|
|
149
168
|
def compile(
|
|
150
169
|
self,
|
|
151
170
|
embedded: bool = True,
|
|
@@ -154,8 +173,14 @@ class AssetTree:
|
|
|
154
173
|
if len(self.nodes) == 0:
|
|
155
174
|
return ""
|
|
156
175
|
|
|
176
|
+
source = ""
|
|
177
|
+
if embedded:
|
|
178
|
+
source += embed("tinyfan", excludes=["codegen.py", "utils/embed.py", "**/__pycache__/*"], minify=False)
|
|
179
|
+
source += (
|
|
180
|
+
"from tinyfan.config import CLI_CONFIG_STATE\n"
|
|
181
|
+
f"CLI_CONFIG_STATE.cli_args = {json.dumps(CLI_CONFIG_STATE.cli_args)}\n"
|
|
182
|
+
)
|
|
157
183
|
if embedded:
|
|
158
|
-
source = embed("tinyfan", excludes=["codegen.py", "utils/embed.py", "**/__pycache__/*"], minify=False)
|
|
159
184
|
root_location = get_root_path(list(self.nodes.values())[0].asset.func)
|
|
160
185
|
source += embed(root_location)
|
|
161
186
|
if not root_location.endswith(".py"):
|
|
@@ -227,6 +252,50 @@ class AssetTree:
|
|
|
227
252
|
"image": DEFAULT_IMAGE,
|
|
228
253
|
"command": ["python"],
|
|
229
254
|
"source": """import os;exec(os.environ['TINYFAN_SOURCE'])""",
|
|
255
|
+
"env": [
|
|
256
|
+
{
|
|
257
|
+
"name": ref._env_name(),
|
|
258
|
+
"valueFrom": {
|
|
259
|
+
"configMapKeyRef": {
|
|
260
|
+
"name": ref.name,
|
|
261
|
+
"key": ref.key,
|
|
262
|
+
}
|
|
263
|
+
},
|
|
264
|
+
}
|
|
265
|
+
for ref in node.refs
|
|
266
|
+
if isinstance(ref, ConfigMapKeyRef)
|
|
267
|
+
]
|
|
268
|
+
+ [
|
|
269
|
+
{
|
|
270
|
+
"name": ref._env_name(),
|
|
271
|
+
"valueFrom": {
|
|
272
|
+
"secretKeyRef": {
|
|
273
|
+
"name": ref.name,
|
|
274
|
+
"key": ref.key,
|
|
275
|
+
}
|
|
276
|
+
},
|
|
277
|
+
}
|
|
278
|
+
for ref in node.refs
|
|
279
|
+
if isinstance(ref, SecretKeyRef)
|
|
280
|
+
],
|
|
281
|
+
"envFrom": [
|
|
282
|
+
{
|
|
283
|
+
"configMapRef": {
|
|
284
|
+
"name": ref.name,
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
for ref in node.refs
|
|
288
|
+
if isinstance(ref, ConfigMapRef)
|
|
289
|
+
]
|
|
290
|
+
+ [
|
|
291
|
+
{
|
|
292
|
+
"secretRef": {
|
|
293
|
+
"name": ref.name,
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
for ref in node.refs
|
|
297
|
+
if isinstance(ref, SecretRef)
|
|
298
|
+
],
|
|
230
299
|
},
|
|
231
300
|
container or {},
|
|
232
301
|
self.flow.container or {},
|
|
@@ -316,11 +385,19 @@ class AssetTree:
|
|
|
316
385
|
|
|
317
386
|
|
|
318
387
|
def codegen(
|
|
388
|
+
flow: Flow | None = None,
|
|
319
389
|
location: str | None = None,
|
|
320
390
|
embedded: bool = False,
|
|
321
391
|
container: ScriptTemplate = {},
|
|
322
392
|
) -> str:
|
|
323
393
|
"""Generate argocd workflow resource as yaml from tinyfan definitions"""
|
|
324
|
-
|
|
394
|
+
CLI_CONFIG_STATE.codegen = True
|
|
395
|
+
if flow:
|
|
396
|
+
result = AssetTree(flow).compile(embedded, container)
|
|
397
|
+
elif location:
|
|
325
398
|
import_all_submodules(location)
|
|
326
|
-
|
|
399
|
+
result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
|
|
400
|
+
else:
|
|
401
|
+
result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
|
|
402
|
+
CLI_CONFIG_STATE.codegen = False
|
|
403
|
+
return result
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from abc import abstractmethod, ABC
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class CliConfigState:
|
|
8
|
+
codegen: bool = False
|
|
9
|
+
cli_args: dict[str, str] = field(default_factory=dict)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
CLI_CONFIG_STATE = CliConfigState()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def cli_arg(name: str, from_env: str | None = None):
|
|
16
|
+
if CLI_CONFIG_STATE.codegen:
|
|
17
|
+
if name not in CLI_CONFIG_STATE.cli_args:
|
|
18
|
+
if from_env is not None:
|
|
19
|
+
val = os.environ.get(from_env)
|
|
20
|
+
if val is None:
|
|
21
|
+
raise Exception(f"neither of cli arg `{name}` nor env var `{from_env}` not setted.")
|
|
22
|
+
return val
|
|
23
|
+
raise Exception(f"cli arg `{name}` is not setted.")
|
|
24
|
+
else:
|
|
25
|
+
return CLI_CONFIG_STATE.cli_args[name]
|
|
26
|
+
else:
|
|
27
|
+
return CLI_CONFIG_STATE.cli_args.get(name)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ConfigValue(ABC):
|
|
31
|
+
@abstractmethod
|
|
32
|
+
def get_value(self) -> str:
|
|
33
|
+
raise Exception(NotImplemented)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Config(ABC):
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def get(self, name: str) -> str:
|
|
39
|
+
raise Exception(NotImplemented)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class ConfigMapRef(Config):
|
|
44
|
+
name: str
|
|
45
|
+
|
|
46
|
+
def get(self, name: str) -> str:
|
|
47
|
+
val = os.environ.get(name)
|
|
48
|
+
if val is None:
|
|
49
|
+
raise Exception(f"Fail to get `{name}` from ConfigMap `{self.name}`")
|
|
50
|
+
else:
|
|
51
|
+
return val
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class SecretRef(Config):
|
|
56
|
+
name: str
|
|
57
|
+
|
|
58
|
+
def get(self, name: str) -> str:
|
|
59
|
+
val = os.environ.get(name)
|
|
60
|
+
if val is None:
|
|
61
|
+
raise Exception(f"Fail to get `{name}` from Secret `{self.name}`")
|
|
62
|
+
else:
|
|
63
|
+
return val
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class ConfigMapKeyRef(ConfigValue):
|
|
68
|
+
name: str
|
|
69
|
+
key: str
|
|
70
|
+
default: str | None = None
|
|
71
|
+
|
|
72
|
+
def _env_name(self) -> str:
|
|
73
|
+
return "TINYFAN_CM__" + self.name + "__" + self.key
|
|
74
|
+
|
|
75
|
+
def get_value(self) -> str:
|
|
76
|
+
val = os.environ.get(self._env_name(), self.default)
|
|
77
|
+
if val is None:
|
|
78
|
+
raise Exception(f"Fail to get `{self.key}` from ConfigMap `{self.name}`")
|
|
79
|
+
else:
|
|
80
|
+
return val
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class SecretKeyRef(ConfigValue):
|
|
85
|
+
name: str
|
|
86
|
+
key: str
|
|
87
|
+
default: str | None = None
|
|
88
|
+
|
|
89
|
+
def _env_name(self) -> str:
|
|
90
|
+
return "TINYFAN_SC__" + self.name + "__" + self.key
|
|
91
|
+
|
|
92
|
+
def get_value(self) -> str:
|
|
93
|
+
val = os.environ.get(self._env_name(), self.default)
|
|
94
|
+
if val is None:
|
|
95
|
+
raise Exception(f"Fail to get `{self.key}` from Secret `{self.name}`")
|
|
96
|
+
else:
|
|
97
|
+
return val
|
|
@@ -3,31 +3,33 @@ from .stores.base import StoreBase
|
|
|
3
3
|
from .stores.naive import NaiveStore
|
|
4
4
|
from .flowrundata import FlowRunData, StoreIdx, UMeta
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
|
-
from typing import Callable, Any, Generic, TypeVar
|
|
6
|
+
from typing import Callable, Any, Generic, TypeVar
|
|
7
|
+
from .resources.base import ResourceBase
|
|
7
8
|
from .argo_typing import ScriptTemplate
|
|
9
|
+
from .config import Config, ConfigValue
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
FLOW_REGISTER = {}
|
|
10
12
|
|
|
11
|
-
Res = TypeVar("Res", bound=Mapping)
|
|
12
13
|
Ret = TypeVar("Ret")
|
|
13
14
|
|
|
14
15
|
DEFAULT_IMAGE = "python:alpine"
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
@dataclass
|
|
18
|
-
class Flow
|
|
19
|
+
class Flow:
|
|
19
20
|
name: str
|
|
20
21
|
tz: str = "UTC"
|
|
21
|
-
assets: dict[str, "Asset[Any,
|
|
22
|
-
resources:
|
|
22
|
+
assets: dict[str, "Asset[Any, Any, Any]"] = field(default_factory=dict)
|
|
23
|
+
resources: dict[str, ResourceBase] | None = None
|
|
23
24
|
container: ScriptTemplate | None = None
|
|
24
25
|
store: StoreBase = field(default_factory=NaiveStore)
|
|
26
|
+
configs: dict[str, Config | ConfigValue | str | None] = field(default_factory=dict)
|
|
25
27
|
|
|
26
28
|
def __post_init__(self):
|
|
27
|
-
|
|
29
|
+
FLOW_REGISTER[self.name] = self
|
|
28
30
|
|
|
29
31
|
|
|
30
|
-
DEFAULT_FLOW: Flow
|
|
32
|
+
DEFAULT_FLOW: Flow = Flow(
|
|
31
33
|
"tinyfan",
|
|
32
34
|
container={
|
|
33
35
|
"image": DEFAULT_IMAGE,
|
|
@@ -36,8 +38,8 @@ DEFAULT_FLOW: Flow[Any] = Flow(
|
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
@dataclass
|
|
39
|
-
class Asset(Generic[Ret,
|
|
40
|
-
flow: Flow
|
|
41
|
+
class Asset(Generic[Ret, UMeta, StoreIdx]):
|
|
42
|
+
flow: Flow
|
|
41
43
|
func: Callable[..., Ret]
|
|
42
44
|
store: StoreBase[Ret, UMeta, StoreIdx]
|
|
43
45
|
schedule: str | None = None
|
|
@@ -62,11 +64,15 @@ class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
|
|
|
62
64
|
mod = inspect.getmodule(self.func)
|
|
63
65
|
if mod and mod.__name__:
|
|
64
66
|
rundata["module_name"] = mod.__name__
|
|
65
|
-
params = {k: v for k, v in rundata.items() if k in func_param_names}
|
|
67
|
+
params: dict[str, object] = {k: v for k, v in rundata.items() if k in func_param_names}
|
|
66
68
|
if self.flow.resources:
|
|
67
|
-
for k, v in self.flow.resources:
|
|
69
|
+
for k, v in self.flow.resources.items():
|
|
68
70
|
if k in func_param_names:
|
|
69
71
|
params[k] = v
|
|
72
|
+
if self.flow.configs:
|
|
73
|
+
for k2, v2 in self.flow.configs.items():
|
|
74
|
+
if k2 in func_param_names:
|
|
75
|
+
params[k2] = v2
|
|
70
76
|
|
|
71
77
|
parent_flowrundatas: dict[str, FlowRunData] = rundata.get("parents") or {}
|
|
72
78
|
for name, prundata in parent_flowrundatas.items():
|
|
@@ -83,11 +89,11 @@ class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
|
|
|
83
89
|
return (ret, rundata)
|
|
84
90
|
|
|
85
91
|
|
|
86
|
-
class AssetFunc(Generic[Ret,
|
|
92
|
+
class AssetFunc(Generic[Ret, UMeta, StoreIdx]):
|
|
87
93
|
func: Callable[..., Ret]
|
|
88
|
-
asset: Asset[Ret,
|
|
94
|
+
asset: Asset[Ret, UMeta, StoreIdx]
|
|
89
95
|
|
|
90
|
-
def __init__(self, func: Callable[..., Ret], asset: Asset[Ret,
|
|
96
|
+
def __init__(self, func: Callable[..., Ret], asset: Asset[Ret, UMeta, StoreIdx]):
|
|
91
97
|
self.func = func
|
|
92
98
|
self.asset = asset
|
|
93
99
|
|
|
@@ -97,16 +103,16 @@ class AssetFunc(Generic[Ret, Res, UMeta, StoreIdx]):
|
|
|
97
103
|
|
|
98
104
|
|
|
99
105
|
def asset(
|
|
100
|
-
flow: Flow
|
|
106
|
+
flow: Flow = DEFAULT_FLOW,
|
|
101
107
|
schedule: str | None = None,
|
|
102
108
|
depends: str | None = None,
|
|
103
109
|
store: StoreBase[Ret, UMeta, StoreIdx] | None = None,
|
|
104
110
|
tz: str | None = None,
|
|
105
111
|
metadata: UMeta | None = None,
|
|
106
112
|
container: ScriptTemplate | None = None,
|
|
107
|
-
) -> Callable[..., AssetFunc[Ret,
|
|
108
|
-
def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret,
|
|
109
|
-
asset = Asset[Ret,
|
|
113
|
+
) -> Callable[..., AssetFunc[Ret, UMeta, StoreIdx]]:
|
|
114
|
+
def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, UMeta, StoreIdx]:
|
|
115
|
+
asset = Asset[Ret, UMeta, StoreIdx](
|
|
110
116
|
flow,
|
|
111
117
|
func=func,
|
|
112
118
|
depends=depends,
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from ..config import Config, ConfigValue
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ResourceBase:
|
|
5
|
+
def get_refs(self) -> set[Config | ConfigValue]:
|
|
6
|
+
refs = set()
|
|
7
|
+
for _, value in vars(self).items():
|
|
8
|
+
if isinstance(value, (Config, ConfigValue)): # or a common base 'Reference'
|
|
9
|
+
refs.add(value)
|
|
10
|
+
elif isinstance(value, ResourceBase):
|
|
11
|
+
refs |= ResourceBase.get_refs(value)
|
|
12
|
+
return refs
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
from abc import abstractmethod
|
|
1
|
+
from abc import abstractmethod, ABC
|
|
2
2
|
from typing import Generic, TypeVar
|
|
3
|
+
from ..resources.base import ResourceBase
|
|
3
4
|
from ..flowrundata import FlowRunData, StoreIdx, UMeta
|
|
4
5
|
|
|
5
6
|
T = TypeVar("T")
|
|
6
7
|
|
|
7
8
|
|
|
8
|
-
class StoreBase(Generic[T, UMeta, StoreIdx]):
|
|
9
|
+
class StoreBase(ABC, ResourceBase, Generic[T, UMeta, StoreIdx]):
|
|
9
10
|
@staticmethod
|
|
10
11
|
@abstractmethod
|
|
11
12
|
def id() -> str:
|
|
File without changes
|
|
@@ -1,24 +1,42 @@
|
|
|
1
1
|
from tinyfan import Flow, asset
|
|
2
|
-
from tinyfan.
|
|
2
|
+
from tinyfan.flow import FLOW_REGISTER
|
|
3
|
+
from tinyfan.codegen import codegen, AssetTree
|
|
3
4
|
from jsonschema.exceptions import ValidationError
|
|
4
|
-
from
|
|
5
|
+
from typing import Generator
|
|
5
6
|
import os
|
|
6
|
-
|
|
7
|
-
from .helpers import gotmpl
|
|
8
|
-
import yaml
|
|
7
|
+
import pytest
|
|
9
8
|
|
|
10
|
-
flow: Flow = Flow("test")
|
|
11
9
|
|
|
10
|
+
# type: ignore
|
|
11
|
+
@pytest.fixture()
|
|
12
|
+
def sample_flow() -> Generator:
|
|
13
|
+
flow: Flow = Flow("test")
|
|
12
14
|
|
|
13
|
-
@asset(flow=flow, schedule="@daily")
|
|
14
|
-
def asset1():
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
@asset(flow=flow, schedule="@daily")
|
|
16
|
+
def asset1(): # type: ignore
|
|
17
|
+
print("asset1 is executed")
|
|
18
|
+
return "hello"
|
|
17
19
|
|
|
20
|
+
@asset(flow=flow)
|
|
21
|
+
def asset2(asset1: str): # type: ignore
|
|
22
|
+
print("asset1 says: " + asset1)
|
|
18
23
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
yield flow
|
|
25
|
+
FLOW_REGISTER.clear()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# type: ignore
|
|
29
|
+
@pytest.fixture()
|
|
30
|
+
def unknown_arg_flow() -> Generator:
|
|
31
|
+
flow: Flow = Flow("test")
|
|
32
|
+
|
|
33
|
+
@asset(flow=flow, schedule="@daily")
|
|
34
|
+
def asset1(unknown: str): # type: ignore
|
|
35
|
+
print("asset1 is executed")
|
|
36
|
+
return "hello"
|
|
37
|
+
|
|
38
|
+
yield flow
|
|
39
|
+
FLOW_REGISTER.clear()
|
|
22
40
|
|
|
23
41
|
|
|
24
42
|
def gotmpl_values(tree: AssetTree, asset_name: str, schedule: str) -> dict:
|
|
@@ -43,6 +61,7 @@ def gotmpl_values(tree: AssetTree, asset_name: str, schedule: str) -> dict:
|
|
|
43
61
|
def assert_code_is_running(code: str):
|
|
44
62
|
# comment out all assertion since templateDefault script override is not working
|
|
45
63
|
return
|
|
64
|
+
"""
|
|
46
65
|
sourcetmpl = next(yaml.load_all(code, yaml.Loader))["spec"]["workflowSpec"]["templates"][0]["script"]["source"]
|
|
47
66
|
tree = AssetTree(flow)
|
|
48
67
|
values = gotmpl_values(tree, "asset1", "@daily")
|
|
@@ -62,15 +81,27 @@ def assert_code_is_running(code: str):
|
|
|
62
81
|
with redirect_stdout(f):
|
|
63
82
|
exec(source)
|
|
64
83
|
assert "asset1 says: hello\n" == f.getvalue()
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_codegen_raise_unknown_asset_args(unknown_arg_flow):
|
|
88
|
+
with pytest.raises(Exception):
|
|
89
|
+
_ = codegen(flow=unknown_arg_flow)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_codegen_flow(validate_crds, sample_flow):
|
|
93
|
+
actual = codegen(flow=sample_flow)
|
|
94
|
+
validate_crds(actual)
|
|
95
|
+
assert_code_is_running(actual)
|
|
65
96
|
|
|
66
97
|
|
|
67
|
-
def
|
|
98
|
+
def test_codegen_implicit(validate_crds, sample_flow):
|
|
68
99
|
actual = codegen()
|
|
69
100
|
validate_crds(actual)
|
|
70
101
|
assert_code_is_running(actual)
|
|
71
102
|
|
|
72
103
|
|
|
73
|
-
def test_embedded_codegen(validate_crds,
|
|
104
|
+
def test_embedded_codegen(validate_crds, sample_flow):
|
|
74
105
|
actual = codegen(embedded=True)
|
|
75
106
|
validate_crds(actual)
|
|
76
107
|
assert_code_is_running(actual)
|
|
File without changes
|
tinyfan-0.0.2/src/tinyfan/cli.py
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import typer
|
|
2
|
-
|
|
3
|
-
from .codegen import codegen
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def tinyfan(
|
|
7
|
-
location: str,
|
|
8
|
-
embedded: bool = False,
|
|
9
|
-
):
|
|
10
|
-
"""
|
|
11
|
-
Generate argocd workflow resource as yaml from tinyfan definitions
|
|
12
|
-
"""
|
|
13
|
-
if location.endswith(".py"):
|
|
14
|
-
print(codegen(location=location, embedded=True))
|
|
15
|
-
else:
|
|
16
|
-
print(codegen(location=location, embedded=embedded))
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def main():
|
|
20
|
-
return typer.run(tinyfan)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if __name__ == "__main__":
|
|
24
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinyfan-0.0.2/tests/codegen_samples/directory → tinyfan-0.1.0/src/tinyfan/utils}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|