tinyfan 0.0.2__tar.gz → 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {tinyfan-0.0.2 → tinyfan-0.1.0}/PKG-INFO +1 -1
  2. tinyfan-0.1.0/examples/configmap.yaml +7 -0
  3. tinyfan-0.1.0/examples/configs.py +31 -0
  4. tinyfan-0.1.0/examples/packaged/Dockerfile +5 -0
  5. tinyfan-0.1.0/examples/packaged/pyproject.toml +13 -0
  6. tinyfan-0.1.0/examples/packaged/src/tinyfan_examples_packaged/asset.py +23 -0
  7. tinyfan-0.1.0/examples/resource.py +41 -0
  8. tinyfan-0.1.0/examples/secret.yaml +7 -0
  9. {tinyfan-0.0.2 → tinyfan-0.1.0}/pyproject.toml +1 -1
  10. tinyfan-0.1.0/src/tinyfan/__init__.py +19 -0
  11. tinyfan-0.1.0/src/tinyfan/cli.py +29 -0
  12. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/codegen.py +82 -5
  13. tinyfan-0.1.0/src/tinyfan/config.py +97 -0
  14. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/flow.py +25 -19
  15. tinyfan-0.1.0/src/tinyfan/resources/base.py +12 -0
  16. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/base.py +3 -2
  17. tinyfan-0.1.0/tests/codegen_samples/directory/__init__.py +0 -0
  18. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/test_codegen.py +46 -15
  19. tinyfan-0.1.0/tests/utils/__init__.py +0 -0
  20. {tinyfan-0.0.2 → tinyfan-0.1.0}/uv.lock +1 -1
  21. tinyfan-0.0.2/examples/realworld/pyproject.toml +0 -7
  22. tinyfan-0.0.2/src/tinyfan/__init__.py +0 -5
  23. tinyfan-0.0.2/src/tinyfan/cli.py +0 -24
  24. {tinyfan-0.0.2 → tinyfan-0.1.0}/.github/workflows/publish.yml +0 -0
  25. {tinyfan-0.0.2 → tinyfan-0.1.0}/.gitignore +0 -0
  26. {tinyfan-0.0.2 → tinyfan-0.1.0}/.python-version +0 -0
  27. {tinyfan-0.0.2 → tinyfan-0.1.0}/README.md +0 -0
  28. {tinyfan-0.0.2 → tinyfan-0.1.0}/examples/getting-started.py +0 -0
  29. {tinyfan-0.0.2/examples/realworld → tinyfan-0.1.0/examples/packaged}/.python-version +0 -0
  30. {tinyfan-0.0.2/examples/realworld → tinyfan-0.1.0/examples/packaged}/README.md +0 -0
  31. {tinyfan-0.0.2/src/tinyfan/stores → tinyfan-0.1.0/examples/packaged/src/tinyfan_examples_packaged}/__init__.py +0 -0
  32. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/argo_typing.py +0 -0
  33. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/flowrundata.py +0 -0
  34. {tinyfan-0.0.2/src/tinyfan/utils → tinyfan-0.1.0/src/tinyfan/resources}/__init__.py +0 -0
  35. {tinyfan-0.0.2/tests → tinyfan-0.1.0/src/tinyfan/stores}/__init__.py +0 -0
  36. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/bigquery.py +0 -0
  37. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/gcs_pickle.py +0 -0
  38. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/stores/naive.py +0 -0
  39. {tinyfan-0.0.2/tests/codegen_samples/directory → tinyfan-0.1.0/src/tinyfan/utils}/__init__.py +0 -0
  40. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/embed.py +0 -0
  41. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/exjson.py +0 -0
  42. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/merge.py +0 -0
  43. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/typing.py +0 -0
  44. {tinyfan-0.0.2 → tinyfan-0.1.0}/src/tinyfan/utils/yaml.py +0 -0
  45. {tinyfan-0.0.2/tests/utils → tinyfan-0.1.0/tests}/__init__.py +0 -0
  46. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/directory/asset.py +0 -0
  47. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/directory/asset2.py +0 -0
  48. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/codegen_samples/singlefile.py +0 -0
  49. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/conftest.py +0 -0
  50. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/helpers.py +0 -0
  51. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/openapi2jsonschema.py +0 -0
  52. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/test_decorators.py +0 -0
  53. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/directory/__init__.py +0 -0
  54. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/directory/dep.py +0 -0
  55. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/embed_samples/singlefile.py +0 -0
  56. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/test_embed.py +0 -0
  57. {tinyfan-0.0.2 → tinyfan-0.1.0}/tests/utils/test_exjson.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tinyfan
3
- Version: 0.0.2
3
+ Version: 0.1.0
4
4
  Summary: Tiny Argo Workflows python devkit
5
5
  Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
6
6
  Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -0,0 +1,7 @@
1
+ apiVersion: v1
2
+ kind: ConfigMap
3
+ metadata:
4
+ name: tinyfan-config-example
5
+ data:
6
+ key: "configmapkeyref"
7
+ otherkey: "otherkeyref"
@@ -0,0 +1,31 @@
1
+ import tinyfan
2
+ from tinyfan import ConfigMapRef, ConfigMapKeyRef, SecretRef, SecretKeyRef, cli_arg
3
+
4
+ flow = tinyfan.Flow(
5
+ name="configs",
6
+ configs={
7
+ "constant": "const",
8
+ "configmap": ConfigMapRef("tinyfan-config-example"),
9
+ "configmapkey": ConfigMapKeyRef("tinyfan-config-example", "key"),
10
+ "secret": SecretRef("tinyfan-config-example"),
11
+ "secretkey": SecretKeyRef("tinyfan-config-example", "secretkey"),
12
+ "cliarg": cli_arg("example"),
13
+ },
14
+ )
15
+
16
+
17
+ @tinyfan.asset(
18
+ schedule="*/1 * * * *",
19
+ flow=flow,
20
+ )
21
+ def trace_configs(
22
+ constant: str,
23
+ cliarg: str,
24
+ configmap: ConfigMapRef,
25
+ configmapkey: ConfigMapKeyRef,
26
+ secret: SecretRef,
27
+ secretkey: SecretKeyRef,
28
+ ):
29
+ print(
30
+ f'{constant} - {cliarg} - {configmap.get("otherkey")} - {configmapkey.get_value()} - {secret.get("othersecretkey")} - {secretkey.get_value()}'
31
+ )
@@ -0,0 +1,5 @@
1
+ FROM python:3.12-slim
2
+
3
+ WORKDIR /opt/app
4
+ COPY . .
5
+ RUN pip install --no-cache-dir .
@@ -0,0 +1,13 @@
1
+ [project]
2
+ name = "tinyfan_examples_packaged"
3
+ version = "0.0.1"
4
+ description = "tinyfan pacakged example"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ dependencies = ["tinyfan"]
8
+
9
+ [build-system]
10
+ requires = ["hatchling"]
11
+ build-backend = "hatchling.build"
12
+
13
+
@@ -0,0 +1,23 @@
1
+ import tinyfan
2
+ from datetime import datetime
3
+
4
+
5
+ @tinyfan.asset(
6
+ schedule="*/1 * * * *", container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"}
7
+ )
8
+ def target() -> str:
9
+ return "world"
10
+
11
+
12
+ @tinyfan.asset(container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"})
13
+ def greeting(target: str):
14
+ print("hello " + target)
15
+
16
+
17
+ @tinyfan.asset(
18
+ schedule="*/1 * * * *", container={"image": "tinyfan-examples-packaged:local", "imagePullPolicy": "Never"}
19
+ )
20
+ def check_variables(ds: str, ts: str, data_interval_start: datetime, data_interval_end: datetime) -> None:
21
+ print(f"ds: {ds}, ts: {ts}")
22
+ print(f"data_interval_start: {data_interval_start}")
23
+ print(f"data_interval_end: {data_interval_end}")
@@ -0,0 +1,41 @@
1
+ import tinyfan
2
+ from tinyfan import ConfigMapKeyRef
3
+
4
+
5
+ class Res(tinyfan.ResourceBase):
6
+ key: ConfigMapKeyRef
7
+
8
+ def __init__(self, key: ConfigMapKeyRef):
9
+ self.key = key
10
+
11
+
12
+ class ResContainer(tinyfan.ResourceBase):
13
+ res: Res
14
+
15
+ def __init__(self, res: Res):
16
+ self.res = res
17
+
18
+
19
+ flow = tinyfan.Flow(
20
+ name="configs",
21
+ resources={
22
+ "res": Res(ConfigMapKeyRef("tinyfan-config-example", "key")),
23
+ "res2": ResContainer(Res(ConfigMapKeyRef("tinyfan-config-example", "key"))),
24
+ },
25
+ )
26
+
27
+
28
+ @tinyfan.asset(
29
+ schedule="*/1 * * * *",
30
+ flow=flow,
31
+ )
32
+ def trace_res(res: Res):
33
+ print(f"{res.key.get_value()}")
34
+
35
+
36
+ @tinyfan.asset(
37
+ schedule="*/1 * * * *",
38
+ flow=flow,
39
+ )
40
+ def trace_res2(res2: ResContainer):
41
+ print(f"{res2.res.key.get_value()}")
@@ -0,0 +1,7 @@
1
+ apiVersion: v1
2
+ kind: Secret
3
+ metadata:
4
+ name: tinyfan-config-example
5
+ stringData:
6
+ secretkey: secretkey # required field for kubernetes.io/basic-auth
7
+ othersecretkey: othersecretkey
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tinyfan"
3
- version = "0.0.2"
3
+ version = "0.1.0"
4
4
  description = 'Tiny Argo Workflows python devkit'
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -0,0 +1,19 @@
1
+ from .flow import Flow, Asset, asset
2
+ from .stores.base import StoreBase
3
+ from .stores.naive import NaiveStore
4
+ from .config import ConfigMapRef, ConfigMapKeyRef, SecretRef, SecretKeyRef, cli_arg
5
+ from .resources.base import ResourceBase
6
+
7
+ __all__ = [
8
+ "Flow",
9
+ "Asset",
10
+ "asset",
11
+ "StoreBase",
12
+ "NaiveStore",
13
+ "ConfigMapRef",
14
+ "ConfigMapKeyRef",
15
+ "SecretRef",
16
+ "SecretKeyRef",
17
+ "cli_arg",
18
+ "ResourceBase",
19
+ ]
@@ -0,0 +1,29 @@
1
+ import typer
2
+
3
+ from typing_extensions import Annotated
4
+ from .codegen import codegen
5
+ from .config import CLI_CONFIG_STATE
6
+
7
+
8
+ def tinyfan(
9
+ location: str,
10
+ embedded: bool | None = None,
11
+ arg: Annotated[list[str] | None, typer.Option(metavar="KEY=VALUE")] = None,
12
+ ):
13
+ """
14
+ Generate argocd workflow resource as yaml from tinyfan definitions
15
+ """
16
+ if arg is not None:
17
+ CLI_CONFIG_STATE.cli_args.update({k: v for (k, v) in (val.split("=") for val in arg)})
18
+ if location.endswith(".py"):
19
+ print(codegen(location=location, embedded=embedded or True))
20
+ else:
21
+ print(codegen(location=location, embedded=embedded or False))
22
+
23
+
24
+ def main():
25
+ return typer.run(tinyfan)
26
+
27
+
28
+ if __name__ == "__main__":
29
+ main()
@@ -1,4 +1,4 @@
1
- from tinyfan.flow import Flow, Asset, FLOW_CATALOG, DEFAULT_IMAGE
1
+ from tinyfan.flow import Flow, Asset, FLOW_REGISTER, DEFAULT_IMAGE
2
2
  import importlib.util
3
3
  from typing import Self
4
4
  from .utils.yaml import dump as yaml_dump, dump_all as yaml_dump_all
@@ -14,6 +14,7 @@ import croniter
14
14
  from .argo_typing import ScriptTemplate
15
15
  from .utils.embed import embed
16
16
  from .utils.merge import deepmerge
17
+ from .config import CLI_CONFIG_STATE, Config, ConfigValue, ConfigMapKeyRef, ConfigMapRef, SecretKeyRef, SecretRef
17
18
 
18
19
  VALID_CRON_REGEXP = r"^(?:(?:(?:(?:\d+,)+\d+|(?:\d+(?:\/|-|#)\d+)|\d+L?|\*(?:\/\d+)?|L(?:-\d+)?|\?|[A-Z]{3}(?:-[A-Z]{3})?) ?){5,7})|(@hourly|@daily|@midnight|@weekly|@monthly|@yearly|@annually)$"
19
20
  VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeeded|Failed|Errored|Skipped|Omitted|Daemoned)\w+)(?!.*(?:&&|\|\|)\s*(?:&&|\|\|))(?!.*!!)[A-Za-z0-9_&|!().\s]+$"
@@ -66,6 +67,7 @@ def import_all_submodules(location: str):
66
67
 
67
68
  class AssetNode:
68
69
  asset: Asset
70
+ refs: list[Config | ConfigValue | str | None]
69
71
  children: list[Self]
70
72
  parents: list[Self]
71
73
 
@@ -130,8 +132,18 @@ class AssetTree:
130
132
  for node in self.nodes.values():
131
133
  sigs = inspect.signature(node.asset.func)
132
134
  func_param_names = list(sigs.parameters.keys())
133
- params_ids = set(name for name in func_param_names if name in self.flow.assets)
135
+ known_names = (
136
+ set(self.flow.assets.keys())
137
+ | set((self.flow.resources or {}).keys())
138
+ | set((self.flow.configs or {}).keys())
139
+ )
140
+ unknown_names = [n for n in func_param_names if n not in known_names]
141
+ if len(unknown_names) > 0:
142
+ raise Exception(
143
+ f"unknown asset args: {', '.join('`' + n + '`' for n in unknown_names)} from asset `{node.asset.name}`"
144
+ )
134
145
 
146
+ params_ids = set(name for name in func_param_names if name in self.flow.assets)
135
147
  if node.asset.depends is not None:
136
148
  depends_ids = set(re.findall(EXTRACT_DEPENDS_REGEXP, node.asset.depends))
137
149
  node.parents = [self.nodes[n] for n in depends_ids.union(params_ids)]
@@ -146,6 +158,13 @@ class AssetTree:
146
158
  p.children.append(node)
147
159
  node.asset.depends = " && ".join(params_ids).replace("_", "-")
148
160
 
161
+ flow_resources = self.flow.resources or {}
162
+ flow_refs = self.flow.configs or {}
163
+ param_resources = [flow_resources[n] for n in func_param_names if n in flow_resources]
164
+ resource_refs = [ref for res in param_resources for ref in res.get_refs()]
165
+ param_refs = [flow_refs[n] for n in func_param_names if n in flow_refs]
166
+ node.refs = list(set(resource_refs + param_refs))
167
+
149
168
  def compile(
150
169
  self,
151
170
  embedded: bool = True,
@@ -154,8 +173,14 @@ class AssetTree:
154
173
  if len(self.nodes) == 0:
155
174
  return ""
156
175
 
176
+ source = ""
177
+ if embedded:
178
+ source += embed("tinyfan", excludes=["codegen.py", "utils/embed.py", "**/__pycache__/*"], minify=False)
179
+ source += (
180
+ "from tinyfan.config import CLI_CONFIG_STATE\n"
181
+ f"CLI_CONFIG_STATE.cli_args = {json.dumps(CLI_CONFIG_STATE.cli_args)}\n"
182
+ )
157
183
  if embedded:
158
- source = embed("tinyfan", excludes=["codegen.py", "utils/embed.py", "**/__pycache__/*"], minify=False)
159
184
  root_location = get_root_path(list(self.nodes.values())[0].asset.func)
160
185
  source += embed(root_location)
161
186
  if not root_location.endswith(".py"):
@@ -227,6 +252,50 @@ class AssetTree:
227
252
  "image": DEFAULT_IMAGE,
228
253
  "command": ["python"],
229
254
  "source": """import os;exec(os.environ['TINYFAN_SOURCE'])""",
255
+ "env": [
256
+ {
257
+ "name": ref._env_name(),
258
+ "valueFrom": {
259
+ "configMapKeyRef": {
260
+ "name": ref.name,
261
+ "key": ref.key,
262
+ }
263
+ },
264
+ }
265
+ for ref in node.refs
266
+ if isinstance(ref, ConfigMapKeyRef)
267
+ ]
268
+ + [
269
+ {
270
+ "name": ref._env_name(),
271
+ "valueFrom": {
272
+ "secretKeyRef": {
273
+ "name": ref.name,
274
+ "key": ref.key,
275
+ }
276
+ },
277
+ }
278
+ for ref in node.refs
279
+ if isinstance(ref, SecretKeyRef)
280
+ ],
281
+ "envFrom": [
282
+ {
283
+ "configMapRef": {
284
+ "name": ref.name,
285
+ }
286
+ }
287
+ for ref in node.refs
288
+ if isinstance(ref, ConfigMapRef)
289
+ ]
290
+ + [
291
+ {
292
+ "secretRef": {
293
+ "name": ref.name,
294
+ }
295
+ }
296
+ for ref in node.refs
297
+ if isinstance(ref, SecretRef)
298
+ ],
230
299
  },
231
300
  container or {},
232
301
  self.flow.container or {},
@@ -316,11 +385,19 @@ class AssetTree:
316
385
 
317
386
 
318
387
  def codegen(
388
+ flow: Flow | None = None,
319
389
  location: str | None = None,
320
390
  embedded: bool = False,
321
391
  container: ScriptTemplate = {},
322
392
  ) -> str:
323
393
  """Generate argocd workflow resource as yaml from tinyfan definitions"""
324
- if location:
394
+ CLI_CONFIG_STATE.codegen = True
395
+ if flow:
396
+ result = AssetTree(flow).compile(embedded, container)
397
+ elif location:
325
398
  import_all_submodules(location)
326
- return "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_CATALOG.values())
399
+ result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
400
+ else:
401
+ result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
402
+ CLI_CONFIG_STATE.codegen = False
403
+ return result
@@ -0,0 +1,97 @@
1
+ from dataclasses import dataclass, field
2
+ from abc import abstractmethod, ABC
3
+ import os
4
+
5
+
6
+ @dataclass
7
+ class CliConfigState:
8
+ codegen: bool = False
9
+ cli_args: dict[str, str] = field(default_factory=dict)
10
+
11
+
12
+ CLI_CONFIG_STATE = CliConfigState()
13
+
14
+
15
+ def cli_arg(name: str, from_env: str | None = None):
16
+ if CLI_CONFIG_STATE.codegen:
17
+ if name not in CLI_CONFIG_STATE.cli_args:
18
+ if from_env is not None:
19
+ val = os.environ.get(from_env)
20
+ if val is None:
21
+ raise Exception(f"neither of cli arg `{name}` nor env var `{from_env}` not setted.")
22
+ return val
23
+ raise Exception(f"cli arg `{name}` is not setted.")
24
+ else:
25
+ return CLI_CONFIG_STATE.cli_args[name]
26
+ else:
27
+ return CLI_CONFIG_STATE.cli_args.get(name)
28
+
29
+
30
+ class ConfigValue(ABC):
31
+ @abstractmethod
32
+ def get_value(self) -> str:
33
+ raise Exception(NotImplemented)
34
+
35
+
36
+ class Config(ABC):
37
+ @abstractmethod
38
+ def get(self, name: str) -> str:
39
+ raise Exception(NotImplemented)
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class ConfigMapRef(Config):
44
+ name: str
45
+
46
+ def get(self, name: str) -> str:
47
+ val = os.environ.get(name)
48
+ if val is None:
49
+ raise Exception(f"Fail to get `{name}` from ConfigMap `{self.name}`")
50
+ else:
51
+ return val
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class SecretRef(Config):
56
+ name: str
57
+
58
+ def get(self, name: str) -> str:
59
+ val = os.environ.get(name)
60
+ if val is None:
61
+ raise Exception(f"Fail to get `{name}` from Secret `{self.name}`")
62
+ else:
63
+ return val
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class ConfigMapKeyRef(ConfigValue):
68
+ name: str
69
+ key: str
70
+ default: str | None = None
71
+
72
+ def _env_name(self) -> str:
73
+ return "TINYFAN_CM__" + self.name + "__" + self.key
74
+
75
+ def get_value(self) -> str:
76
+ val = os.environ.get(self._env_name(), self.default)
77
+ if val is None:
78
+ raise Exception(f"Fail to get `{self.key}` from ConfigMap `{self.name}`")
79
+ else:
80
+ return val
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class SecretKeyRef(ConfigValue):
85
+ name: str
86
+ key: str
87
+ default: str | None = None
88
+
89
+ def _env_name(self) -> str:
90
+ return "TINYFAN_SC__" + self.name + "__" + self.key
91
+
92
+ def get_value(self) -> str:
93
+ val = os.environ.get(self._env_name(), self.default)
94
+ if val is None:
95
+ raise Exception(f"Fail to get `{self.key}` from Secret `{self.name}`")
96
+ else:
97
+ return val
@@ -3,31 +3,33 @@ from .stores.base import StoreBase
3
3
  from .stores.naive import NaiveStore
4
4
  from .flowrundata import FlowRunData, StoreIdx, UMeta
5
5
  from dataclasses import dataclass, field
6
- from typing import Callable, Any, Generic, TypeVar, Mapping
6
+ from typing import Callable, Any, Generic, TypeVar
7
+ from .resources.base import ResourceBase
7
8
  from .argo_typing import ScriptTemplate
9
+ from .config import Config, ConfigValue
8
10
 
9
- FLOW_CATALOG = {}
11
+ FLOW_REGISTER = {}
10
12
 
11
- Res = TypeVar("Res", bound=Mapping)
12
13
  Ret = TypeVar("Ret")
13
14
 
14
15
  DEFAULT_IMAGE = "python:alpine"
15
16
 
16
17
 
17
18
  @dataclass
18
- class Flow(Generic[Res]):
19
+ class Flow:
19
20
  name: str
20
21
  tz: str = "UTC"
21
- assets: dict[str, "Asset[Any, Res, Any, Any]"] = field(default_factory=dict)
22
- resources: Res | None = None
22
+ assets: dict[str, "Asset[Any, Any, Any]"] = field(default_factory=dict)
23
+ resources: dict[str, ResourceBase] | None = None
23
24
  container: ScriptTemplate | None = None
24
25
  store: StoreBase = field(default_factory=NaiveStore)
26
+ configs: dict[str, Config | ConfigValue | str | None] = field(default_factory=dict)
25
27
 
26
28
  def __post_init__(self):
27
- FLOW_CATALOG[self.name] = self
29
+ FLOW_REGISTER[self.name] = self
28
30
 
29
31
 
30
- DEFAULT_FLOW: Flow[Any] = Flow(
32
+ DEFAULT_FLOW: Flow = Flow(
31
33
  "tinyfan",
32
34
  container={
33
35
  "image": DEFAULT_IMAGE,
@@ -36,8 +38,8 @@ DEFAULT_FLOW: Flow[Any] = Flow(
36
38
 
37
39
 
38
40
  @dataclass
39
- class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
40
- flow: Flow[Res]
41
+ class Asset(Generic[Ret, UMeta, StoreIdx]):
42
+ flow: Flow
41
43
  func: Callable[..., Ret]
42
44
  store: StoreBase[Ret, UMeta, StoreIdx]
43
45
  schedule: str | None = None
@@ -62,11 +64,15 @@ class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
62
64
  mod = inspect.getmodule(self.func)
63
65
  if mod and mod.__name__:
64
66
  rundata["module_name"] = mod.__name__
65
- params = {k: v for k, v in rundata.items() if k in func_param_names}
67
+ params: dict[str, object] = {k: v for k, v in rundata.items() if k in func_param_names}
66
68
  if self.flow.resources:
67
- for k, v in self.flow.resources:
69
+ for k, v in self.flow.resources.items():
68
70
  if k in func_param_names:
69
71
  params[k] = v
72
+ if self.flow.configs:
73
+ for k2, v2 in self.flow.configs.items():
74
+ if k2 in func_param_names:
75
+ params[k2] = v2
70
76
 
71
77
  parent_flowrundatas: dict[str, FlowRunData] = rundata.get("parents") or {}
72
78
  for name, prundata in parent_flowrundatas.items():
@@ -83,11 +89,11 @@ class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
83
89
  return (ret, rundata)
84
90
 
85
91
 
86
- class AssetFunc(Generic[Ret, Res, UMeta, StoreIdx]):
92
+ class AssetFunc(Generic[Ret, UMeta, StoreIdx]):
87
93
  func: Callable[..., Ret]
88
- asset: Asset[Ret, Res, UMeta, StoreIdx]
94
+ asset: Asset[Ret, UMeta, StoreIdx]
89
95
 
90
- def __init__(self, func: Callable[..., Ret], asset: Asset[Ret, Res, UMeta, StoreIdx]):
96
+ def __init__(self, func: Callable[..., Ret], asset: Asset[Ret, UMeta, StoreIdx]):
91
97
  self.func = func
92
98
  self.asset = asset
93
99
 
@@ -97,16 +103,16 @@ class AssetFunc(Generic[Ret, Res, UMeta, StoreIdx]):
97
103
 
98
104
 
99
105
  def asset(
100
- flow: Flow[Res] = DEFAULT_FLOW,
106
+ flow: Flow = DEFAULT_FLOW,
101
107
  schedule: str | None = None,
102
108
  depends: str | None = None,
103
109
  store: StoreBase[Ret, UMeta, StoreIdx] | None = None,
104
110
  tz: str | None = None,
105
111
  metadata: UMeta | None = None,
106
112
  container: ScriptTemplate | None = None,
107
- ) -> Callable[..., AssetFunc[Ret, Res, UMeta, StoreIdx]]:
108
- def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, Res, UMeta, StoreIdx]:
109
- asset = Asset[Ret, Res, UMeta, StoreIdx](
113
+ ) -> Callable[..., AssetFunc[Ret, UMeta, StoreIdx]]:
114
+ def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, UMeta, StoreIdx]:
115
+ asset = Asset[Ret, UMeta, StoreIdx](
110
116
  flow,
111
117
  func=func,
112
118
  depends=depends,
@@ -0,0 +1,12 @@
1
+ from ..config import Config, ConfigValue
2
+
3
+
4
+ class ResourceBase:
5
+ def get_refs(self) -> set[Config | ConfigValue]:
6
+ refs = set()
7
+ for _, value in vars(self).items():
8
+ if isinstance(value, (Config, ConfigValue)): # or a common base 'Reference'
9
+ refs.add(value)
10
+ elif isinstance(value, ResourceBase):
11
+ refs |= ResourceBase.get_refs(value)
12
+ return refs
@@ -1,11 +1,12 @@
1
- from abc import abstractmethod
1
+ from abc import abstractmethod, ABC
2
2
  from typing import Generic, TypeVar
3
+ from ..resources.base import ResourceBase
3
4
  from ..flowrundata import FlowRunData, StoreIdx, UMeta
4
5
 
5
6
  T = TypeVar("T")
6
7
 
7
8
 
8
- class StoreBase(Generic[T, UMeta, StoreIdx]):
9
+ class StoreBase(ABC, ResourceBase, Generic[T, UMeta, StoreIdx]):
9
10
  @staticmethod
10
11
  @abstractmethod
11
12
  def id() -> str:
@@ -1,24 +1,42 @@
1
1
  from tinyfan import Flow, asset
2
- from tinyfan.codegen import codegen, AssetTree, RUNDATA_FILE_PATH
2
+ from tinyfan.flow import FLOW_REGISTER
3
+ from tinyfan.codegen import codegen, AssetTree
3
4
  from jsonschema.exceptions import ValidationError
4
- from io import StringIO
5
+ from typing import Generator
5
6
  import os
6
- from contextlib import redirect_stdout
7
- from .helpers import gotmpl
8
- import yaml
7
+ import pytest
9
8
 
10
- flow: Flow = Flow("test")
11
9
 
10
+ # type: ignore
11
+ @pytest.fixture()
12
+ def sample_flow() -> Generator:
13
+ flow: Flow = Flow("test")
12
14
 
13
- @asset(flow=flow, schedule="@daily")
14
- def asset1():
15
- print("asset1 is executed")
16
- return "hello"
15
+ @asset(flow=flow, schedule="@daily")
16
+ def asset1(): # type: ignore
17
+ print("asset1 is executed")
18
+ return "hello"
17
19
 
20
+ @asset(flow=flow)
21
+ def asset2(asset1: str): # type: ignore
22
+ print("asset1 says: " + asset1)
18
23
 
19
- @asset(flow=flow)
20
- def asset2(asset1: str):
21
- print("asset1 says: " + asset1)
24
+ yield flow
25
+ FLOW_REGISTER.clear()
26
+
27
+
28
+ # type: ignore
29
+ @pytest.fixture()
30
+ def unknown_arg_flow() -> Generator:
31
+ flow: Flow = Flow("test")
32
+
33
+ @asset(flow=flow, schedule="@daily")
34
+ def asset1(unknown: str): # type: ignore
35
+ print("asset1 is executed")
36
+ return "hello"
37
+
38
+ yield flow
39
+ FLOW_REGISTER.clear()
22
40
 
23
41
 
24
42
  def gotmpl_values(tree: AssetTree, asset_name: str, schedule: str) -> dict:
@@ -43,6 +61,7 @@ def gotmpl_values(tree: AssetTree, asset_name: str, schedule: str) -> dict:
43
61
  def assert_code_is_running(code: str):
44
62
  # comment out all assertion since templateDefault script override is not working
45
63
  return
64
+ """
46
65
  sourcetmpl = next(yaml.load_all(code, yaml.Loader))["spec"]["workflowSpec"]["templates"][0]["script"]["source"]
47
66
  tree = AssetTree(flow)
48
67
  values = gotmpl_values(tree, "asset1", "@daily")
@@ -62,15 +81,27 @@ def assert_code_is_running(code: str):
62
81
  with redirect_stdout(f):
63
82
  exec(source)
64
83
  assert "asset1 says: hello\n" == f.getvalue()
84
+ """
85
+
86
+
87
+ def test_codegen_raise_unknown_asset_args(unknown_arg_flow):
88
+ with pytest.raises(Exception):
89
+ _ = codegen(flow=unknown_arg_flow)
90
+
91
+
92
+ def test_codegen_flow(validate_crds, sample_flow):
93
+ actual = codegen(flow=sample_flow)
94
+ validate_crds(actual)
95
+ assert_code_is_running(actual)
65
96
 
66
97
 
67
- def test_codegen(validate_crds, assert_manifests):
98
+ def test_codegen_implicit(validate_crds, sample_flow):
68
99
  actual = codegen()
69
100
  validate_crds(actual)
70
101
  assert_code_is_running(actual)
71
102
 
72
103
 
73
- def test_embedded_codegen(validate_crds, assert_manifests):
104
+ def test_embedded_codegen(validate_crds, sample_flow):
74
105
  actual = codegen(embedded=True)
75
106
  validate_crds(actual)
76
107
  assert_code_is_running(actual)
File without changes
@@ -774,7 +774,7 @@ wheels = [
774
774
 
775
775
  [[package]]
776
776
  name = "tinyfan"
777
- version = "0.0.2"
777
+ version = "0.1.0"
778
778
  source = { editable = "." }
779
779
  dependencies = [
780
780
  { name = "croniter" },
@@ -1,7 +0,0 @@
1
- [project]
2
- name = "code"
3
- version = "0.0.1"
4
- description = "vtown"
5
- readme = "README.md"
6
- requires-python = ">=3.10"
7
- dependencies = []
@@ -1,5 +0,0 @@
1
- from .flow import Flow, Asset, asset
2
- from .stores.base import StoreBase
3
- from .stores.naive import NaiveStore
4
-
5
- __all__ = ["Flow", "Asset", "asset", "StoreBase", "NaiveStore"]
@@ -1,24 +0,0 @@
1
- import typer
2
-
3
- from .codegen import codegen
4
-
5
-
6
- def tinyfan(
7
- location: str,
8
- embedded: bool = False,
9
- ):
10
- """
11
- Generate argocd workflow resource as yaml from tinyfan definitions
12
- """
13
- if location.endswith(".py"):
14
- print(codegen(location=location, embedded=True))
15
- else:
16
- print(codegen(location=location, embedded=embedded))
17
-
18
-
19
- def main():
20
- return typer.run(tinyfan)
21
-
22
-
23
- if __name__ == "__main__":
24
- main()
File without changes
File without changes
File without changes
File without changes
File without changes