tinyfan 0.2.3__tar.gz → 0.2.5__tar.gz

{tinyfan-0.2.3 → tinyfan-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.2.3
+Version: 0.2.5
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues

{tinyfan-0.2.3 → tinyfan-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinyfan"
-version = "0.2.3"
+version = "0.2.5"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"

{tinyfan-0.2.3 → tinyfan-0.2.5}/src/tinyfan/argo_typing.py

@@ -93,6 +93,22 @@ class EnvVar(TypedDict):
     valueFrom: NotRequired[EnvVarSource]
 
 
+class ConfigMapEnvSource(TypedDict, total=False):
+    name: str
+    optional: bool
+
+
+class SecretEnvSource(TypedDict, total=False):
+    name: str
+    optional: bool
+
+
+class EnvFromSource(TypedDict, total=False):
+    configMapRef: ConfigMapEnvSource
+    secretRef: SecretEnvSource
+    prefix: str
+
+
 class Lifecycle(TypedDict, total=False):
     postStart: LifecycleHandler
     preStop: LifecycleHandler
@@ -183,7 +199,7 @@ class ScriptTemplate(TypedDict, total=False):
     args: list[str]
     command: list[str]
     env: list[EnvVar]
-    envFrom: list[EnvVar]
+    envFrom: list[EnvFromSource]
     image: str
     imagePullPolicy: str
     lifecycle: Lifecycle

{tinyfan-0.2.3 → tinyfan-0.2.5}/src/tinyfan/codegen.py

@@ -117,11 +117,11 @@ class AssetNode:
 
     def relatives(self, result: set[Self] | None = None) -> set[Self]:
         if result is None:
-            result = set[Self]([self])
-        for o in self.parents + self.children:
-            if o not in result:
-                result.add(o)
-                o.relatives(result)
+            result = {self}
+        for node in self.parents + self.children:
+            if node not in result:
+                result.add(node)
+                node.relatives(result)
         return result
 
 
@@ -142,18 +142,23 @@ class AssetTree:
                 | set(inspect.get_annotations(FlowRunData).keys())
             )
             unknown_names = [n for n in func_param_names if n not in known_names]
-            if len(unknown_names) > 0:
+            if unknown_names:
                 raise Exception(
                     f"unknown asset args: {', '.join('`' + n + '`' for n in unknown_names)} from asset `{node.asset.name}`"
                 )
 
             params_ids = set(name for name in func_param_names if name in self.flow.assets)
             if node.asset.depends is not None:
+                assert re.match(VALID_DEPENDS_REGEXP, node.asset.depends)
                 depends_ids = set(re.findall(EXTRACT_DEPENDS_REGEXP, node.asset.depends))
                 node.parents = [self.nodes[n] for n in depends_ids.union(params_ids)]
-                node.asset.depends = f"({node.asset.depends}) && {' && '.join(params_ids - depends_ids)}".replace(
-                    "_", "-"
-                )
+                new_ids = params_ids - depends_ids
+                if new_ids:
+                    if len(depends_ids) > 1:
+                        node.asset.depends = f"({node.asset.depends}) && {' && '.join(new_ids)}".replace("_", "-")
+                    else:
+                        node.asset.depends = f"{node.asset.depends} && {' && '.join(new_ids)}".replace("_", "-")
+                    assert re.match(VALID_DEPENDS_REGEXP, node.asset.depends)
                 for p in node.parents:
                     p.children.append(node)
             else:
@@ -189,10 +194,9 @@ class AssetTree:
 
         from .dagviz import dagviz
 
-        relatives_by_schedules = self.relatives_by_schedules()
         g: nx.DiGraph = nx.DiGraph()
         g.add_node(self.flow.name, bullet="■")
-        for (tz, schedule), relatives in relatives_by_schedules.items():
+        for (tz, schedule), relatives in self.relatives_by_schedules().items():
             schedule_node_name = schedule + f" ({tz})"
             g.add_node(schedule_node_name, bullet="○")
             g.add_edge(self.flow.name, schedule_node_name)

{tinyfan-0.2.3 → tinyfan-0.2.5}/src/tinyfan/resources/base.py

@@ -1,10 +1,11 @@
 from ..config import Config, ConfigValue
+import inspect
 
 
 class ResourceBase:
     def get_refs(self) -> set[Config | ConfigValue]:
         refs = set()
-        for _, value in vars(self).items():
+        for _, value in inspect.getmembers(self):
             if isinstance(value, (Config, ConfigValue)):  # or a common base 'Reference'
                 refs.add(value)
             elif isinstance(value, ResourceBase):

{tinyfan-0.2.3 → tinyfan-0.2.5}/tests/test_codegen.py

@@ -21,6 +21,14 @@ def sample_flow() -> Generator:
     def asset2(asset1: str):  # type: ignore
         print("asset1 says: " + asset1)
 
+    @asset(flow=flow, depends="asset1 && asset2")
+    def asset3(asset1: str):  # type: ignore
+        print("asset1 says: " + asset1)
+
+    @asset(flow=flow, depends="asset2")
+    def asset4(asset1: str):  # type: ignore
+        print("asset1 says: " + asset1)
+
     yield flow
     FLOW_REGISTER.clear()
 
@@ -132,9 +140,18 @@ def test_codegen_directory(validate_crds):
 def test_rundata_data_interval(sample_flow):
     tree = AssetTree(sample_flow)
     node = list(tree.nodes.values())[0]
-    # rundatatmpl = node.rundatatmpl("0 * * * * *")
     daily = node.rundatatmpl("@daily")
     assert (
         daily
         == """{"ds": "{{=sprig.date('2006-01-02', sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime)))}}", "ts": "{{=sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))}}", "data_interval_start": {"__tinyfan_datetime__": "{{=sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))}}"}, "data_interval_end": {"__tinyfan_datetime__": "{{workflow.scheduledTime}}"}, "parents": {}, "asset_name": "asset1", "flow_name": "test", "module_name": "tests.test_codegen"}"""
     )
+
+
+def test_depends(sample_flow):
+    tree = AssetTree(sample_flow)
+    assert [node.asset.depends for node in tree.nodes.values()] == [
+        "",
+        "asset1",
+        "asset1 && asset2",
+        "asset2 && asset1",
+    ]

tinyfan-0.2.5/tests/test_config.py

@@ -0,0 +1,60 @@
+from tinyfan import Flow, asset, ResourceBase, ConfigMapRef, ConfigMapKeyRef, SecretRef, SecretKeyRef
+from tinyfan.flow import FLOW_REGISTER
+from tinyfan.codegen import codegen
+from typing import Generator
+from unittest import mock
+import pytest
+import os
+
+
+# type: ignore
+@pytest.fixture()
+def sample_flow() -> Generator:
+    class Resource(ResourceBase):
+        cm = ConfigMapRef("cm")
+        cm_key = ConfigMapKeyRef("cm", "key")
+        cm_default = ConfigMapKeyRef("cm", "non_key", "default")
+        sec = SecretRef("secret")
+        sec_key = SecretKeyRef("secret", "key")
+        sec_default = SecretKeyRef("secret", "non_key", "default")
+
+    flow: Flow = Flow(
+        "test",
+        resources={
+            "res": Resource(),
+        },
+    )
+
+    @asset(flow=flow, schedule="@daily")
+    def test_asset(res: Resource):  # type: ignore
+        return (
+            res.cm.get("key"),
+            res.cm_key.get_value(),
+            res.cm_default.get_value(),
+            res.sec.get("key"),
+            res.sec_key.get_value(),
+            res.sec_default.get_value(),
+        )
+
+    yield flow
+    FLOW_REGISTER.clear()
+
+
+# HACK IT: skip it at this stage. too tired.
+def test_resource_config_codgen(validate_crds, sample_flow):
+    res = codegen(flow=sample_flow)
+    validate_crds(res)
+
+
+def test_resource_config_run(sample_flow: Flow):
+    env = {
+        "TINYFAN_CM__cm__key": "cmkey",
+        "TINYFAN_SC__secret__key": "sekey",
+        "key": "key",
+    }
+    with mock.patch.dict(os.environ, env):
+        _ = codegen(flow=sample_flow)
+        asset = list(sample_flow.assets.values())[0]
+        actual, _ = asset.run()
+        expected = ("key", "cmkey", "default", "key", "sekey", "default")
+        assert actual == expected

{tinyfan-0.2.3 → tinyfan-0.2.5}/uv.lock

@@ -784,7 +784,7 @@ wheels = [
 
 [[package]]
 name = "tinyfan"
-version = "0.2.3"
+version = "0.2.5"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },