tinyfan 0.2.8__tar.gz → 0.3.0__tar.gz

{tinyfan-0.2.8 → tinyfan-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.2.8
+Version: 0.3.0
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -9,14 +9,12 @@ Author-email: eunchuldev <eunchulsong@gmail.com>
 Keywords: argo-workflow,pipeline,tinyfan,workflow
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: croniter>=3.0.3
-Requires-Dist: pyyaml>=6.0.2
-Requires-Dist: typer-slim>=0.15.1
+Requires-Dist: croniter>=6
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: typer-slim>=0.24.0
 Provides-Extra: all
 Requires-Dist: google-cloud-bigquery[pandas]; extra == 'all'
 Requires-Dist: google-cloud-storage; extra == 'all'
@@ -40,6 +38,7 @@ Requires-Dist: pytest>=8.3.3; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: types-croniter; extra == 'dev'
 Requires-Dist: types-jsonschema; extra == 'dev'
+Requires-Dist: types-networkx; extra == 'dev'
 Requires-Dist: types-pyyaml; extra == 'dev'
 Provides-Extra: gcs
 Requires-Dist: google-cloud-storage; extra == 'gcs'
@@ -99,7 +98,7 @@ def greeting(world: str):
 ```shell
 # Apply the changes to argo workflow
 
-tinyfan template main.py | kubectl apply -f -
+tinyfan main.py | kubectl apply -f -
 ```
 
 # Real World Example (still tiny though!)

{tinyfan-0.2.8 → tinyfan-0.3.0}/README.md

@@ -47,7 +47,7 @@ def greeting(world: str):
 ```shell
 # Apply the changes to argo workflow
 
-tinyfan template main.py | kubectl apply -f -
+tinyfan main.py | kubectl apply -f -
 ```
 
 # Real World Example (still tiny though!)

{tinyfan-0.2.8 → tinyfan-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinyfan"
-version = "0.2.8"
+version = "0.3.0"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"
@@ -9,12 +9,10 @@ authors = [{ name = "eunchuldev", email = "eunchulsong@gmail.com" }]
 classifiers = [
   "Operating System :: OS Independent",
   "License :: OSI Approved :: MIT License",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Typing :: Typed",
 ]
-dependencies = ["croniter>=3.0.3", "pyyaml>=6.0.2", "typer-slim>=0.15.1"]
+dependencies = ["croniter>=6", "pyyaml>=6.0.3", "typer-slim>=0.24.0"]
 
 [project.scripts]
 tinyfan = "tinyfan.cli:main"
@@ -34,6 +32,7 @@ dev = [
   "types-PyYAML",
   "types-jsonschema",
   "types-croniter",
+  "types-networkx",
   "tinyfan[all]",
 ]
 
@@ -53,7 +52,7 @@ addopts = ["--import-mode=importlib"]
 exclude = ['build']
 
 [[tool.mypy.overrides]]
-module = ["lark.*", "jsonpatch.*"]
+module = ["lark.*", "jsonpatch.*", "google.*"]
 ignore_missing_imports = true
 
 [tool.ruff]

{tinyfan-0.2.8 → tinyfan-0.3.0}/src/tinyfan/argo_typing.py

@@ -1,4 +1,4 @@
-from typing import TypedDict, NotRequired
+from typing import Any, TypedDict, NotRequired
 
 
 class ExecAction(TypedDict):
@@ -194,6 +194,16 @@ class VolumeMount(TypedDict, total=False):
     subPathExpr: str
 
 
+class TemplateInputs(TypedDict, total=False):
+    artifacts: list[dict[str, Any]]
+    parameters: list[dict[str, Any]]
+
+
+class TemplateOutputs(TypedDict, total=False):
+    artifacts: list[dict[str, Any]]
+    parameters: list[dict[str, Any]]
+
+
 # Ref: https://argo-workflows.readthedocs.io/en/latest/fields/#scripttemplate
 class ScriptTemplate(TypedDict, total=False):
     args: list[str]

{tinyfan-0.2.8 → tinyfan-0.3.0}/src/tinyfan/codegen.py

@@ -1,4 +1,5 @@
 from tinyfan.flow import Flow, Asset, FLOW_REGISTER, DEFAULT_IMAGE
+import copy
 import importlib.util
 from typing import Self
 from .utils.yaml import dump as yaml_dump, dump_all as yaml_dump_all
@@ -11,7 +12,7 @@ import os
 import pkgutil
 import datetime
 import croniter
-from .argo_typing import ScriptTemplate
+from .argo_typing import ScriptTemplate, TemplateInputs, TemplateOutputs
 from .utils.embed import embed
 from .utils.merge import deepmerge, dropnone
 from .config import CLI_CONFIG_STATE, Config, ConfigValue, ConfigMapKeyRef, ConfigMapRef, SecretKeyRef, SecretRef
@@ -23,6 +24,34 @@ VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeede
 EXTRACT_DEPENDS_REGEXP = r"(?<!\.)\b([A-Za-z0-9_]+)\b"
 
 RUNDATA_FILE_PATH = "/tmp/tinyfan/rundata.json"
+INTERNAL_INPUT_PARAMETERS = {"rundata", "asset_name", "moduledata"}
+INTERNAL_OUTPUT_PARAMETERS = {"rundata"}
+
+
+def merge_template_io(template_io: dict, user_io: TemplateInputs | TemplateOutputs | None, reserved_parameter_names: set[str]) -> dict:
+    result = copy.deepcopy(template_io)
+    if user_io is None:
+        return result
+    for key, value in user_io.items():
+        if key in ("parameters", "artifacts"):
+            if not isinstance(value, list):
+                raise TypeError(f"asset {key} must be a list")
+            if key == "parameters":
+                conflicts = [
+                    param.get("name")
+                    for param in value
+                    if isinstance(param, dict) and param.get("name") in reserved_parameter_names
+                ]
+                if conflicts:
+                    names = ", ".join(f"`{name}`" for name in conflicts)
+                    raise ValueError(f"asset parameters cannot redefine tinyfan internal parameter(s): {names}")
+            result.setdefault(key, [])
+            result[key].extend(copy.deepcopy(value))
+        elif isinstance(result.get(key), dict) and isinstance(value, dict):
+            result[key] = deepmerge(result[key], copy.deepcopy(value))
+        else:
+            result[key] = copy.deepcopy(value)
+    return result
 
 
 def brackets_balanced(code: str) -> bool:
@@ -257,7 +286,7 @@ class AssetTree:
                     "generateName": f"""{self.flow.name}-{encode_schedule_to_k8sname(tz, schedule)}-""",
                 },
                 "spec": {
-                    "schedule": schedule,
+                    "schedules": [schedule],
                     "timezone": tz,
                     "workflowSpec": {
                         "entrypoint": "flow",
@@ -364,32 +393,40 @@ class AssetTree:
                                 "synchronization": {
                                     "mutexes": [{"name": f"{self.flow.name}-{node.asset.name.replace('_', '-')}"}]
                                 },
-                                "inputs": {
-                                    "parameters": [
-                                        {
-                                            "name": "rundata",
-                                            "value": "{{= nil }}",
-                                        },
-                                        {
-                                            "name": "asset_name",
-                                            "value": "{{= nil }}",
-                                        },
-                                        {
-                                            "name": "moduledata",
-                                            "value": "{{= nil }}",
-                                        },
-                                    ]
-                                },
-                                "outputs": {
-                                    "parameters": [
-                                        {
-                                            "name": "rundata",
-                                            "valueFrom": {
-                                                "path": RUNDATA_FILE_PATH,
+                                "inputs": merge_template_io(
+                                    {
+                                        "parameters": [
+                                            {
+                                                "name": "rundata",
+                                                "value": "{{= nil }}",
                                             },
-                                        }
-                                    ],
-                                },
+                                            {
+                                                "name": "asset_name",
+                                                "value": "{{= nil }}",
+                                            },
+                                            {
+                                                "name": "moduledata",
+                                                "value": "{{= nil }}",
+                                            },
+                                        ]
+                                    },
+                                    node.asset.inputs,
+                                    INTERNAL_INPUT_PARAMETERS,
+                                ),
+                                "outputs": merge_template_io(
+                                    {
+                                        "parameters": [
+                                            {
+                                                "name": "rundata",
+                                                "valueFrom": {
+                                                    "path": RUNDATA_FILE_PATH,
+                                                },
+                                            }
+                                        ],
+                                    },
+                                    node.asset.outputs,
+                                    INTERNAL_OUTPUT_PARAMETERS,
+                                ),
                             }
                             for node in relatives
                         ]
@@ -461,17 +498,19 @@ def codegen(
     serviceAccountName: str | None = None,
 ) -> str:
     """Generate argocd workflow resource as yaml from tinyfan definitions"""
-    CLI_CONFIG_STATE.codegen = True
-    if flow:
-        result = AssetTree(flow).compile(embedded, container, serviceAccountName)
-    elif location:
-        import_all_submodules(location)
-        result = "\n---\n".join(
-            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
-        )
-    else:
-        result = "\n---\n".join(
-            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
-        )
-    CLI_CONFIG_STATE.codegen = False
+    try:
+        CLI_CONFIG_STATE.codegen = True
+        if flow:
+            result = AssetTree(flow).compile(embedded, container, serviceAccountName)
+        elif location:
+            import_all_submodules(location)
+            result = "\n---\n".join(
+                AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+            )
+        else:
+            result = "\n---\n".join(
+                AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+            )
+    finally:
+        CLI_CONFIG_STATE.codegen = False
     return result

{tinyfan-0.2.8 → tinyfan-0.3.0}/src/tinyfan/flow.py

@@ -6,7 +6,7 @@ from .flowrundata import FlowRunData, StoreIdx, UMeta
 from dataclasses import dataclass, field
 from typing import Callable, Any, Generic, TypeVar, Awaitable, Coroutine, cast
 from .resources.base import ResourceBase
-from .argo_typing import ScriptTemplate
+from .argo_typing import ScriptTemplate, TemplateInputs, TemplateOutputs
 from .config import Config, ConfigValue
 
 FLOW_REGISTER = {}
@@ -51,6 +51,8 @@ class Asset(Generic[Ret, UMeta, StoreIdx]):
     name: str = field(init=False)
     container: ScriptTemplate | None = None
     serviceAccountName: str | None = None
+    inputs: TemplateInputs | None = None
+    outputs: TemplateOutputs | None = None
 
     def __post_init__(self):
         self.name = self.func.__name__
@@ -81,7 +83,10 @@ class Asset(Generic[Ret, UMeta, StoreIdx]):
         for name, prundata in parent_flowrundatas.items():
             parent = self.flow.assets[name]
             index = prundata.get("store_entry_idx", None)
-            data = index and parent.store.retrieve(index, source_rundata=prundata, target_rundata=rundata)
+            if index is not None:
+                data = parent.store.retrieve(index, source_rundata=prundata, target_rundata=rundata)
+            else:
+                data = None
             if name in func_param_names:
                 params[name] = data
         ret = self.func(**params)
@@ -118,6 +123,8 @@ def asset(
     metadata: UMeta | None = None,
     container: ScriptTemplate | None = None,
     serviceAccountName: str | None = None,
+    inputs: TemplateInputs | None = None,
+    outputs: TemplateOutputs | None = None,
 ) -> Callable[..., AssetFunc[Ret, UMeta, StoreIdx]]:
     def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, UMeta, StoreIdx]:
         asset = Asset[Ret, UMeta, StoreIdx](
@@ -130,6 +137,8 @@ def asset(
             metadata=metadata,
             container=container,
             serviceAccountName=serviceAccountName,
+            inputs=inputs,
+            outputs=outputs,
         )
         return AssetFunc(func, asset)
 

tinyfan-0.3.0/src/tinyfan/stores/gcs_pickle.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import pickle
+import re
+import uuid
+from collections.abc import Mapping
+from typing import Any, NotRequired, TypedDict
+
+from ..config import ConfigValue
+from .base import FlowRunData, StoreBase
+
+
+class GcsPickleStoreIndex(TypedDict):
+    bucket: str
+    name: str
+    generation: NotRequired[int]
+
+
+class GcsPickleStoreMetadata(TypedDict, total=False):
+    blob_name: str
+    prefix: str
+
+
+ConfigurableString = str | ConfigValue
+
+
+_GCS_EXTRA_ERROR = "GCS extra not installed. Please install with `pip install tinyfan[gcs]`."
+
+
+def _new_storage_client(client_kwargs: Mapping[str, Any]) -> Any:
+    try:
+        from google.cloud import storage
+    except ImportError as exc:
+        raise ImportError(_GCS_EXTRA_ERROR) from exc
+
+    return storage.Client(**client_kwargs)
+
+
+def _resolve(value: ConfigurableString) -> str:
+    if isinstance(value, ConfigValue):
+        return value.get_value()
+    return value
+
+
+def _path_part(value: object, fallback: str) -> str:
+    text = str(value if value is not None else fallback).strip()
+    text = re.sub(r"[^A-Za-z0-9._=-]+", "-", text)
+    return text.strip(".-") or fallback
+
+
+def _join_blob_name(*parts: str) -> str:
+    return "/".join(part.strip("/") for part in parts if part and part.strip("/"))
+
+
+def _run_part(rundata: FlowRunData[Any, Any]) -> str:
+    for key in ("data_interval_start", "ts", "ds"):
+        value = rundata.get(key)
+        if value is None:
+            continue
+        if hasattr(value, "isoformat"):
+            value = value.isoformat()
+        return _path_part(value, "manual")
+    return "manual"
+
+
+class GcsPickleStore(StoreBase[object, GcsPickleStoreMetadata, GcsPickleStoreIndex]):
+    bucket: ConfigurableString
+    prefix: ConfigurableString
+
+    def __init__(
+        self,
+        bucket: ConfigurableString,
+        prefix: ConfigurableString = "tinyfan",
+        *,
+        client: Any | None = None,
+        client_kwargs: Mapping[str, Any] | None = None,
+        pickle_protocol: int = pickle.HIGHEST_PROTOCOL,
+    ) -> None:
+        self.bucket = bucket
+        self.prefix = prefix
+        self._client = client
+        self._client_kwargs = dict(client_kwargs or {})
+        self.pickle_protocol = pickle_protocol
+
+    @staticmethod
+    def id() -> str:
+        return "tinyfan.gcs_pickle"
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            self._client = _new_storage_client(self._client_kwargs)
+        return self._client
+
+    def _bucket_name(self) -> str:
+        bucket = _resolve(self.bucket).strip()
+        if not bucket:
+            raise ValueError("GCS bucket name must not be empty.")
+        return bucket
+
+    def _blob_name(self, rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex]) -> str:
+        metadata = rundata.get("metadata")
+        if isinstance(metadata, Mapping):
+            blob_name = metadata.get("blob_name")
+            if blob_name is not None:
+                blob_name = str(blob_name).strip("/")
+                if not blob_name:
+                    raise ValueError("GCS blob_name metadata must not be empty.")
+                return blob_name
+
+            prefix = str(metadata.get("prefix") or _resolve(self.prefix))
+        else:
+            prefix = _resolve(self.prefix)
+
+        flow_name = _path_part(rundata.get("flow_name"), "flow")
+        asset_name = _path_part(rundata.get("asset_name"), "asset")
+        run_id = _run_part(rundata)
+        filename = f"{uuid.uuid4().hex}.pickle"
+        return _join_blob_name(prefix, flow_name, asset_name, run_id, filename)
+
+    def store(
+        self,
+        value: object,
+        rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex],
+    ) -> GcsPickleStoreIndex:
+        bucket_name = self._bucket_name()
+        blob = self._get_client().bucket(bucket_name).blob(self._blob_name(rundata))
+        data = pickle.dumps(value, protocol=self.pickle_protocol)
+        blob.upload_from_string(data, content_type="application/octet-stream")
+
+        index: GcsPickleStoreIndex = {
+            "bucket": bucket_name,
+            "name": blob.name,
+        }
+        generation = getattr(blob, "generation", None)
+        if generation is not None:
+            index["generation"] = int(generation)
+        return index
+
+    def retrieve(
+        self,
+        index: GcsPickleStoreIndex,
+        source_rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex],
+        target_rundata: FlowRunData,
+    ) -> object:
+        del source_rundata, target_rundata
+
+        bucket = self._get_client().bucket(index["bucket"])
+        blob = bucket.blob(index["name"], generation=index.get("generation"))
+        return pickle.loads(blob.download_as_bytes())

{tinyfan-0.2.8 → tinyfan-0.3.0}/src/tinyfan/utils/embed.py

@@ -54,7 +54,7 @@ def zip_directory(
         files -= set().union(*map(path.glob, excludes))
         for file_path in map(str, files):
             arcname = os.path.relpath(file_path, start=parent_dir)
-            with open(file_path, "r") as f:
+            with open(file_path, "r", encoding="utf-8") as f:
                 try:
                     source = f.read()
                 except Exception as e:
@@ -67,7 +67,7 @@ def zip_directory(
 
 
 def embed_singlefile(path: str) -> str:
-    with open(path, "r") as f:
+    with open(path, "r", encoding="utf-8") as f:
         sourcecode = f.read()
     return sourcecode
 

{tinyfan-0.2.8 → tinyfan-0.3.0}/tests/test_codegen.py

@@ -5,6 +5,7 @@ from jsonschema.exceptions import ValidationError
 from typing import Generator
 import os
 import pytest
+import yaml
 
 
 # type: ignore
@@ -103,6 +104,38 @@ def test_codegen_flow(validate_crds, sample_flow):
     assert_code_is_running(actual)
 
 
+def test_codegen_asset_inputs_outputs(validate_crds):
+    flow: Flow = Flow("test-io")
+
+    @asset(
+        flow=flow,
+        schedule="@daily",
+        inputs={"parameters": [{"name": "message", "value": "hello"}]},
+        outputs={"parameters": [{"name": "summary", "valueFrom": {"path": "/tmp/summary.txt"}}]},
+    )
+    def asset1():  # type: ignore
+        return "hello"
+
+    try:
+        actual = codegen(flow=flow)
+        validate_crds(actual)
+        manifest = next(yaml.load_all(actual, yaml.Loader))
+        template = next(
+            template
+            for template in manifest["spec"]["workflowSpec"]["templates"]
+            if template["name"] == "asset1"
+        )
+        input_params = {param["name"]: param for param in template["inputs"]["parameters"]}
+        output_params = {param["name"]: param for param in template["outputs"]["parameters"]}
+
+        assert input_params["message"] == {"name": "message", "value": "hello"}
+        assert {"rundata", "asset_name", "moduledata"}.issubset(input_params.keys())
+        assert output_params["summary"] == {"name": "summary", "valueFrom": {"path": "/tmp/summary.txt"}}
+        assert output_params["rundata"] == {"name": "rundata", "valueFrom": {"path": "/tmp/tinyfan/rundata.json"}}
+    finally:
+        FLOW_REGISTER.clear()
+
+
 def test_codegen_implicit(validate_crds, sample_flow):
     actual = codegen()
     validate_crds(actual)

tinyfan-0.3.0/tests/test_gcs_pickle_store.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from tinyfan.stores.gcs_pickle import GcsPickleStore
+
+
+class FakeBlob:
+    def __init__(self, name: str, generation: int | None = None) -> None:
+        self.name = name
+        self.generation = generation
+        self.content_type: str | None = None
+        self.data: bytes | None = None
+
+    def upload_from_string(self, data: bytes, content_type: str) -> None:
+        self.data = data
+        self.content_type = content_type
+        self.generation = (self.generation or 0) + 1
+
+    def download_as_bytes(self) -> bytes:
+        assert self.data is not None
+        return self.data
+
+
+class FakeBucket:
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.blobs: dict[str, FakeBlob] = {}
+
+    def blob(self, name: str, generation: int | None = None) -> FakeBlob:
+        blob = self.blobs.setdefault(name, FakeBlob(name, generation))
+        blob.generation = generation or blob.generation
+        return blob
+
+
+class FakeClient:
+    def __init__(self) -> None:
+        self.buckets: dict[str, FakeBucket] = {}
+
+    def bucket(self, name: str) -> FakeBucket:
+        return self.buckets.setdefault(name, FakeBucket(name))
+
+
+def test_gcs_pickle_store_roundtrip() -> None:
+    client = FakeClient()
+    store = GcsPickleStore("test-bucket", prefix="runs", client=client)
+    value = {"hello": ["world", 1]}
+    rundata = {
+        "flow_name": "flow",
+        "asset_name": "asset",
+        "ds": "2026-05-31",
+        "data_interval_start": datetime(2026, 5, 31, 12, 30, tzinfo=UTC),
+    }
+
+    index = store.store(value, rundata)
+
+    assert index["bucket"] == "test-bucket"
+    assert index["name"].startswith("runs/flow/asset/2026-05-31T12-30-00-00-00/")
+    assert index["name"].endswith(".pickle")
+    assert index["generation"] == 1
+    assert client.bucket("test-bucket").blob(index["name"]).content_type == "application/octet-stream"
+    assert store.retrieve(index, source_rundata=rundata, target_rundata={}) == value
+
+
+def test_gcs_pickle_store_uses_metadata_blob_name() -> None:
+    store = GcsPickleStore("test-bucket", client=FakeClient())
+    rundata = {"metadata": {"blob_name": "/custom/path/value.pkl"}}
+
+    index = store.store("value", rundata)
+
+    assert index["name"] == "custom/path/value.pkl"