tinyfan 0.2.9__tar.gz → 0.4.0__tar.gz

{tinyfan-0.2.9 → tinyfan-0.4.0}/.gitignore

@@ -8,3 +8,7 @@ wheels/
 
 # Virtual environments
 .venv
+
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/

{tinyfan-0.2.9 → tinyfan-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.2.9
+Version: 0.4.0
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -9,14 +9,12 @@ Author-email: eunchuldev <eunchulsong@gmail.com>
 Keywords: argo-workflow,pipeline,tinyfan,workflow
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: croniter>=6.0.0
+Requires-Dist: croniter>=6
 Requires-Dist: pyyaml>=6.0.3
-Requires-Dist: typer-slim>=0.20.0
+Requires-Dist: typer-slim>=0.24.0
 Provides-Extra: all
 Requires-Dist: google-cloud-bigquery[pandas]; extra == 'all'
 Requires-Dist: google-cloud-storage; extra == 'all'
@@ -103,6 +101,18 @@ def greeting(world: str):
 tinyfan main.py | kubectl apply -f -
 ```
 
+For a package, pass its directory or import name. Tinyfan statically discovers
+modules containing `@asset`, so unrelated submodules are not executed during
+manifest generation. Modules imported by an asset module (for example, a shared
+`Flow` definition) are loaded normally.
+
+```shell
+tinyfan src/my_package
+```
+
+If a project registers assets dynamically or wraps `@asset` in another decorator,
+use `--import-all` to opt into the previous eager-import behavior.
+
 # Real World Example (still tiny though!)
 
 Comming soon

{tinyfan-0.2.9 → tinyfan-0.4.0}/README.md

@@ -50,6 +50,18 @@ def greeting(world: str):
 tinyfan main.py | kubectl apply -f -
 ```
 
+For a package, pass its directory or import name. Tinyfan statically discovers
+modules containing `@asset`, so unrelated submodules are not executed during
+manifest generation. Modules imported by an asset module (for example, a shared
+`Flow` definition) are loaded normally.
+
+```shell
+tinyfan src/my_package
+```
+
+If a project registers assets dynamically or wraps `@asset` in another decorator,
+use `--import-all` to opt into the previous eager-import behavior.
+
 # Real World Example (still tiny though!)
 
 Comming soon

{tinyfan-0.2.9 → tinyfan-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinyfan"
-version = "0.2.9"
+version = "0.4.0"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"
@@ -9,12 +9,10 @@ authors = [{ name = "eunchuldev", email = "eunchulsong@gmail.com" }]
 classifiers = [
   "Operating System :: OS Independent",
   "License :: OSI Approved :: MIT License",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Typing :: Typed",
 ]
-dependencies = ["croniter>=6.0.0", "pyyaml>=6.0.3", "typer-slim>=0.20.0"]
+dependencies = ["croniter>=6", "pyyaml>=6.0.3", "typer-slim>=0.24.0"]
 
 [project.scripts]
 tinyfan = "tinyfan.cli:main"
@@ -54,7 +52,7 @@ addopts = ["--import-mode=importlib"]
 exclude = ['build']
 
 [[tool.mypy.overrides]]
-module = ["lark.*", "jsonpatch.*"]
+module = ["lark.*", "jsonpatch.*", "google.*"]
 ignore_missing_imports = true
 
 [tool.ruff]

{tinyfan-0.2.9 → tinyfan-0.4.0}/src/tinyfan/argo_typing.py

@@ -1,4 +1,4 @@
-from typing import TypedDict, NotRequired
+from typing import Any, TypedDict, NotRequired
 
 
 class ExecAction(TypedDict):
@@ -194,6 +194,16 @@ class VolumeMount(TypedDict, total=False):
     subPathExpr: str
 
 
+class TemplateInputs(TypedDict, total=False):
+    artifacts: list[dict[str, Any]]
+    parameters: list[dict[str, Any]]
+
+
+class TemplateOutputs(TypedDict, total=False):
+    artifacts: list[dict[str, Any]]
+    parameters: list[dict[str, Any]]
+
+
 # Ref: https://argo-workflows.readthedocs.io/en/latest/fields/#scripttemplate
 class ScriptTemplate(TypedDict, total=False):
     args: list[str]

{tinyfan-0.2.9 → tinyfan-0.4.0}/src/tinyfan/cli.py

@@ -9,6 +9,10 @@ def tinyfan(
     location: str,
     embedded: bool | None = None,
     arg: Annotated[list[str] | None, typer.Option(metavar="KEY=VALUE")] = None,
+    import_all: Annotated[
+        bool,
+        typer.Option(help="Import every submodule instead of discovering modules with @asset statically."),
+    ] = False,
 ):
     """
     Generate argocd workflow resource as yaml from tinyfan definitions
@@ -16,9 +20,9 @@ def tinyfan(
     if arg is not None:
         CLI_CONFIG_STATE.cli_args.update({k: v for (k, v) in (val.split("=") for val in arg)})
     if location.endswith(".py"):
-        print(codegen(location=location, embedded=embedded or True))
+        print(codegen(location=location, embedded=embedded or True, import_all=import_all))
     else:
-        print(codegen(location=location, embedded=embedded or False))
+        print(codegen(location=location, embedded=embedded or False, import_all=import_all))
 
 
 def main():

{tinyfan-0.2.9 → tinyfan-0.4.0}/src/tinyfan/codegen.py

@@ -1,4 +1,6 @@
 from tinyfan.flow import Flow, Asset, FLOW_REGISTER, DEFAULT_IMAGE
+import ast
+import copy
 import importlib.util
 from typing import Self
 from .utils.yaml import dump as yaml_dump, dump_all as yaml_dump_all
@@ -8,10 +10,11 @@ import sys
 import re
 import json
 import os
-import pkgutil
+from pathlib import Path
+import tokenize
 import datetime
 import croniter
-from .argo_typing import ScriptTemplate
+from .argo_typing import ScriptTemplate, TemplateInputs, TemplateOutputs
 from .utils.embed import embed
 from .utils.merge import deepmerge, dropnone
 from .config import CLI_CONFIG_STATE, Config, ConfigValue, ConfigMapKeyRef, ConfigMapRef, SecretKeyRef, SecretRef
@@ -23,6 +26,36 @@ VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeede
 EXTRACT_DEPENDS_REGEXP = r"(?<!\.)\b([A-Za-z0-9_]+)\b"
 
 RUNDATA_FILE_PATH = "/tmp/tinyfan/rundata.json"
+INTERNAL_INPUT_PARAMETERS = {"rundata", "asset_name", "moduledata"}
+INTERNAL_OUTPUT_PARAMETERS = {"rundata"}
+
+
+def merge_template_io(
+    template_io: dict, user_io: TemplateInputs | TemplateOutputs | None, reserved_parameter_names: set[str]
+) -> dict:
+    result = copy.deepcopy(template_io)
+    if user_io is None:
+        return result
+    for key, value in user_io.items():
+        if key in ("parameters", "artifacts"):
+            if not isinstance(value, list):
+                raise TypeError(f"asset {key} must be a list")
+            if key == "parameters":
+                conflicts = [
+                    param.get("name")
+                    for param in value
+                    if isinstance(param, dict) and param.get("name") in reserved_parameter_names
+                ]
+                if conflicts:
+                    names = ", ".join(f"`{name}`" for name in conflicts)
+                    raise ValueError(f"asset parameters cannot redefine tinyfan internal parameter(s): {names}")
+            result.setdefault(key, [])
+            result[key].extend(copy.deepcopy(value))
+        elif isinstance(result.get(key), dict) and isinstance(value, dict):
+            result[key] = deepmerge(result[key], copy.deepcopy(value))
+        else:
+            result[key] = copy.deepcopy(value)
+    return result
 
 
 def brackets_balanced(code: str) -> bool:
@@ -46,7 +79,85 @@ def encode_schedule_to_k8sname(tz: str, cron: str) -> str:
     return f"{tz}|{cron}".lower().translate(str.maketrans("* ,-/_+|", "a-cds-p-", "@"))[:58]
 
 
-def import_all_submodules(location: str):
+def _defines_tinyfan_asset(source_path: Path) -> bool:
+    """Return whether a module has a statically recognizable Tinyfan asset."""
+    with tokenize.open(source_path) as source_file:
+        tree = ast.parse(source_file.read(), filename=str(source_path))
+
+    asset_names: set[str] = set()
+    tinyfan_namespaces: set[str] = set()
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for imported in node.names:
+                if imported.name == "tinyfan":
+                    tinyfan_namespaces.add(imported.asname or "tinyfan")
+                elif imported.name == "tinyfan.flow":
+                    tinyfan_namespaces.add(imported.asname or "tinyfan")
+        elif isinstance(node, ast.ImportFrom) and node.module in {"tinyfan", "tinyfan.flow"}:
+            for imported in node.names:
+                if imported.name == "asset":
+                    asset_names.add(imported.asname or imported.name)
+                elif imported.name == "*":
+                    asset_names.add("asset")
+
+    def is_asset_decorator(decorator: ast.expr) -> bool:
+        while isinstance(decorator, ast.Call):
+            decorator = decorator.func
+        if isinstance(decorator, ast.Name):
+            return decorator.id in asset_names
+
+        names: list[str] = []
+        while isinstance(decorator, ast.Attribute):
+            names.append(decorator.attr)
+            decorator = decorator.value
+        if isinstance(decorator, ast.Name):
+            names.append(decorator.id)
+        names.reverse()
+        return len(names) >= 2 and names[0] in tinyfan_namespaces and names[-1] == "asset"
+
+    class AssetVisitor(ast.NodeVisitor):
+        found = False
+
+        def _visit_definition(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
+            if any(is_asset_decorator(decorator) for decorator in node.decorator_list):
+                self.found = True
+
+        def visit_FunctionDef(self, node: ast.FunctionDef):
+            self._visit_definition(node)
+
+        def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+            self._visit_definition(node)
+
+        def visit_ClassDef(self, node: ast.ClassDef):
+            self._visit_definition(node)
+            if not self.found:
+                self.generic_visit(node)
+
+    visitor = AssetVisitor()
+    visitor.visit(tree)
+    return visitor.found
+
+
+def _package_source_modules(package) -> list[tuple[str, Path]]:
+    modules: list[tuple[str, Path]] = []
+    for package_path in package.__path__:
+        root = Path(package_path)
+        for source_path in root.rglob("*.py"):
+            relative_path = source_path.relative_to(root)
+            if source_path.name == "__init__.py":
+                module_parts = relative_path.parts[:-1]
+            else:
+                module_parts = relative_path.with_suffix("").parts
+            if not module_parts:
+                # Importing the root package has already executed its __init__.py.
+                continue
+            modules.append((".".join((package.__name__, *module_parts)), source_path))
+    return modules
+
+
+def import_asset_modules(location: str, import_all: bool = False):
+    """Import asset-bearing modules from a module name, package directory, or file."""
     try:
         package = importlib.import_module(location)
     except ModuleNotFoundError:
@@ -61,9 +172,16 @@ def import_all_submodules(location: str):
         package = importlib.util.module_from_spec(spec)
         sys.modules[spec.name] = package
         spec.loader.exec_module(package)
-    if package.__name__ != "__tinyfan_repo__":
-        for _, name, _ in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
-            importlib.import_module(name)
+    if package.__name__ == "__tinyfan_repo__" or not hasattr(package, "__path__"):
+        return
+    for module_name, source_path in _package_source_modules(package):
+        if import_all or _defines_tinyfan_asset(source_path):
+            importlib.import_module(module_name)
+
+
+def import_all_submodules(location: str):
+    """Import every submodule, preserving the eager discovery API used before 0.3.0."""
+    import_asset_modules(location, import_all=True)
 
 
 def assetname2template(name: str):
@@ -364,32 +482,40 @@ class AssetTree:
                                 "synchronization": {
                                     "mutexes": [{"name": f"{self.flow.name}-{node.asset.name.replace('_', '-')}"}]
                                 },
-                                "inputs": {
-                                    "parameters": [
-                                        {
-                                            "name": "rundata",
-                                            "value": "{{= nil }}",
-                                        },
-                                        {
-                                            "name": "asset_name",
-                                            "value": "{{= nil }}",
-                                        },
-                                        {
-                                            "name": "moduledata",
-                                            "value": "{{= nil }}",
-                                        },
-                                    ]
-                                },
-                                "outputs": {
-                                    "parameters": [
-                                        {
-                                            "name": "rundata",
-                                            "valueFrom": {
-                                                "path": RUNDATA_FILE_PATH,
+                                "inputs": merge_template_io(
+                                    {
+                                        "parameters": [
+                                            {
+                                                "name": "rundata",
+                                                "value": "{{= nil }}",
                                             },
-                                        }
-                                    ],
-                                },
+                                            {
+                                                "name": "asset_name",
+                                                "value": "{{= nil }}",
+                                            },
+                                            {
+                                                "name": "moduledata",
+                                                "value": "{{= nil }}",
+                                            },
+                                        ]
+                                    },
+                                    node.asset.inputs,
+                                    INTERNAL_INPUT_PARAMETERS,
+                                ),
+                                "outputs": merge_template_io(
+                                    {
+                                        "parameters": [
+                                            {
+                                                "name": "rundata",
+                                                "valueFrom": {
+                                                    "path": RUNDATA_FILE_PATH,
+                                                },
+                                            }
+                                        ],
+                                    },
+                                    node.asset.outputs,
+                                    INTERNAL_OUTPUT_PARAMETERS,
+                                ),
                             }
                             for node in relatives
                         ]
@@ -459,19 +585,22 @@ def codegen(
     embedded: bool = False,
     container: ScriptTemplate = {},
     serviceAccountName: str | None = None,
+    import_all: bool = False,
 ) -> str:
     """Generate argocd workflow resource as yaml from tinyfan definitions"""
-    CLI_CONFIG_STATE.codegen = True
-    if flow:
-        result = AssetTree(flow).compile(embedded, container, serviceAccountName)
-    elif location:
-        import_all_submodules(location)
-        result = "\n---\n".join(
-            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
-        )
-    else:
-        result = "\n---\n".join(
-            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
-        )
-    CLI_CONFIG_STATE.codegen = False
+    try:
+        CLI_CONFIG_STATE.codegen = True
+        if flow:
+            result = AssetTree(flow).compile(embedded, container, serviceAccountName)
+        elif location:
+            import_asset_modules(location, import_all=import_all)
+            result = "\n---\n".join(
+                AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+            )
+        else:
+            result = "\n---\n".join(
+                AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+            )
+    finally:
+        CLI_CONFIG_STATE.codegen = False
     return result

{tinyfan-0.2.9 → tinyfan-0.4.0}/src/tinyfan/flow.py

@@ -6,7 +6,7 @@ from .flowrundata import FlowRunData, StoreIdx, UMeta
 from dataclasses import dataclass, field
 from typing import Callable, Any, Generic, TypeVar, Awaitable, Coroutine, cast
 from .resources.base import ResourceBase
-from .argo_typing import ScriptTemplate
+from .argo_typing import ScriptTemplate, TemplateInputs, TemplateOutputs
 from .config import Config, ConfigValue
 
 FLOW_REGISTER = {}
@@ -51,6 +51,8 @@ class Asset(Generic[Ret, UMeta, StoreIdx]):
     name: str = field(init=False)
     container: ScriptTemplate | None = None
     serviceAccountName: str | None = None
+    inputs: TemplateInputs | None = None
+    outputs: TemplateOutputs | None = None
 
     def __post_init__(self):
         self.name = self.func.__name__
@@ -81,7 +83,10 @@ class Asset(Generic[Ret, UMeta, StoreIdx]):
         for name, prundata in parent_flowrundatas.items():
             parent = self.flow.assets[name]
             index = prundata.get("store_entry_idx", None)
-            data = index and parent.store.retrieve(index, source_rundata=prundata, target_rundata=rundata)
+            if index is not None:
+                data = parent.store.retrieve(index, source_rundata=prundata, target_rundata=rundata)
+            else:
+                data = None
             if name in func_param_names:
                 params[name] = data
         ret = self.func(**params)
@@ -118,6 +123,8 @@ def asset(
     metadata: UMeta | None = None,
     container: ScriptTemplate | None = None,
     serviceAccountName: str | None = None,
+    inputs: TemplateInputs | None = None,
+    outputs: TemplateOutputs | None = None,
 ) -> Callable[..., AssetFunc[Ret, UMeta, StoreIdx]]:
     def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, UMeta, StoreIdx]:
         asset = Asset[Ret, UMeta, StoreIdx](
@@ -130,6 +137,8 @@ def asset(
             metadata=metadata,
             container=container,
             serviceAccountName=serviceAccountName,
+            inputs=inputs,
+            outputs=outputs,
         )
         return AssetFunc(func, asset)
 

tinyfan-0.4.0/src/tinyfan/stores/gcs_pickle.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import pickle
+import re
+import uuid
+from collections.abc import Mapping
+from typing import Any, NotRequired, TypedDict
+
+from ..config import ConfigValue
+from .base import FlowRunData, StoreBase
+
+
+class GcsPickleStoreIndex(TypedDict):
+    bucket: str
+    name: str
+    generation: NotRequired[int]
+
+
+class GcsPickleStoreMetadata(TypedDict, total=False):
+    blob_name: str
+    prefix: str
+
+
+ConfigurableString = str | ConfigValue
+
+
+_GCS_EXTRA_ERROR = "GCS extra not installed. Please install with `pip install tinyfan[gcs]`."
+
+
+def _new_storage_client(client_kwargs: Mapping[str, Any]) -> Any:
+    try:
+        from google.cloud import storage
+    except ImportError as exc:
+        raise ImportError(_GCS_EXTRA_ERROR) from exc
+
+    return storage.Client(**client_kwargs)
+
+
+def _resolve(value: ConfigurableString) -> str:
+    if isinstance(value, ConfigValue):
+        return value.get_value()
+    return value
+
+
+def _path_part(value: object, fallback: str) -> str:
+    text = str(value if value is not None else fallback).strip()
+    text = re.sub(r"[^A-Za-z0-9._=-]+", "-", text)
+    return text.strip(".-") or fallback
+
+
+def _join_blob_name(*parts: str) -> str:
+    return "/".join(part.strip("/") for part in parts if part and part.strip("/"))
+
+
+def _run_part(rundata: FlowRunData[Any, Any]) -> str:
+    for key in ("data_interval_start", "ts", "ds"):
+        value = rundata.get(key)
+        if value is None:
+            continue
+        if hasattr(value, "isoformat"):
+            value = value.isoformat()
+        return _path_part(value, "manual")
+    return "manual"
+
+
+class GcsPickleStore(StoreBase[object, GcsPickleStoreMetadata, GcsPickleStoreIndex]):
+    bucket: ConfigurableString
+    prefix: ConfigurableString
+
+    def __init__(
+        self,
+        bucket: ConfigurableString,
+        prefix: ConfigurableString = "tinyfan",
+        *,
+        client: Any | None = None,
+        client_kwargs: Mapping[str, Any] | None = None,
+        pickle_protocol: int = pickle.HIGHEST_PROTOCOL,
+    ) -> None:
+        self.bucket = bucket
+        self.prefix = prefix
+        self._client = client
+        self._client_kwargs = dict(client_kwargs or {})
+        self.pickle_protocol = pickle_protocol
+
+    @staticmethod
+    def id() -> str:
+        return "tinyfan.gcs_pickle"
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            self._client = _new_storage_client(self._client_kwargs)
+        return self._client
+
+    def _bucket_name(self) -> str:
+        bucket = _resolve(self.bucket).strip()
+        if not bucket:
+            raise ValueError("GCS bucket name must not be empty.")
+        return bucket
+
+    def _blob_name(self, rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex]) -> str:
+        metadata = rundata.get("metadata")
+        if isinstance(metadata, Mapping):
+            blob_name = metadata.get("blob_name")
+            if blob_name is not None:
+                blob_name = str(blob_name).strip("/")
+                if not blob_name:
+                    raise ValueError("GCS blob_name metadata must not be empty.")
+                return blob_name
+
+            prefix = str(metadata.get("prefix") or _resolve(self.prefix))
+        else:
+            prefix = _resolve(self.prefix)
+
+        flow_name = _path_part(rundata.get("flow_name"), "flow")
+        asset_name = _path_part(rundata.get("asset_name"), "asset")
+        run_id = _run_part(rundata)
+        filename = f"{uuid.uuid4().hex}.pickle"
+        return _join_blob_name(prefix, flow_name, asset_name, run_id, filename)
+
+    def store(
+        self,
+        value: object,
+        rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex],
+    ) -> GcsPickleStoreIndex:
+        bucket_name = self._bucket_name()
+        blob = self._get_client().bucket(bucket_name).blob(self._blob_name(rundata))
+        data = pickle.dumps(value, protocol=self.pickle_protocol)
+        blob.upload_from_string(data, content_type="application/octet-stream")
+
+        index: GcsPickleStoreIndex = {
+            "bucket": bucket_name,
+            "name": blob.name,
+        }
+        generation = getattr(blob, "generation", None)
+        if generation is not None:
+            index["generation"] = int(generation)
+        return index
+
+    def retrieve(
+        self,
+        index: GcsPickleStoreIndex,
+        source_rundata: FlowRunData[GcsPickleStoreMetadata, GcsPickleStoreIndex],
+        target_rundata: FlowRunData,
+    ) -> object:
+        del source_rundata, target_rundata
+
+        bucket = self._get_client().bucket(index["bucket"])
+        blob = bucket.blob(index["name"], generation=index.get("generation"))
+        return pickle.loads(blob.download_as_bytes())

{tinyfan-0.2.9 → tinyfan-0.4.0}/src/tinyfan/utils/embed.py

@@ -54,7 +54,7 @@ def zip_directory(
         files -= set().union(*map(path.glob, excludes))
         for file_path in map(str, files):
             arcname = os.path.relpath(file_path, start=parent_dir)
-            with open(file_path, "r") as f:
+            with open(file_path, "r", encoding="utf-8") as f:
                 try:
                     source = f.read()
                 except Exception as e:
@@ -67,7 +67,7 @@ def zip_directory(
 
 
 def embed_singlefile(path: str) -> str:
-    with open(path, "r") as f:
+    with open(path, "r", encoding="utf-8") as f:
         sourcecode = f.read()
     return sourcecode
 

{tinyfan-0.2.9 → tinyfan-0.4.0}/tests/test_codegen.py

@@ -4,7 +4,9 @@ from tinyfan.codegen import codegen, AssetTree
 from jsonschema.exceptions import ValidationError
 from typing import Generator
 import os
+import sys
 import pytest
+import yaml
 
 
 # type: ignore
@@ -103,6 +105,36 @@ def test_codegen_flow(validate_crds, sample_flow):
     assert_code_is_running(actual)
 
 
+def test_codegen_asset_inputs_outputs(validate_crds):
+    flow: Flow = Flow("test-io")
+
+    @asset(
+        flow=flow,
+        schedule="@daily",
+        inputs={"parameters": [{"name": "message", "value": "hello"}]},
+        outputs={"parameters": [{"name": "summary", "valueFrom": {"path": "/tmp/summary.txt"}}]},
+    )
+    def asset1():  # type: ignore
+        return "hello"
+
+    try:
+        actual = codegen(flow=flow)
+        validate_crds(actual)
+        manifest = next(yaml.load_all(actual, yaml.Loader))
+        template = next(
+            template for template in manifest["spec"]["workflowSpec"]["templates"] if template["name"] == "asset1"
+        )
+        input_params = {param["name"]: param for param in template["inputs"]["parameters"]}
+        output_params = {param["name"]: param for param in template["outputs"]["parameters"]}
+
+        assert input_params["message"] == {"name": "message", "value": "hello"}
+        assert {"rundata", "asset_name", "moduledata"}.issubset(input_params.keys())
+        assert output_params["summary"] == {"name": "summary", "valueFrom": {"path": "/tmp/summary.txt"}}
+        assert output_params["rundata"] == {"name": "rundata", "valueFrom": {"path": "/tmp/tinyfan/rundata.json"}}
+    finally:
+        FLOW_REGISTER.clear()
+
+
 def test_codegen_implicit(validate_crds, sample_flow):
     actual = codegen()
     validate_crds(actual)
@@ -137,6 +169,74 @@ def test_codegen_directory(validate_crds):
     assert_code_is_running(actual)
 
 
+def test_codegen_directory_only_imports_asset_modules(tmp_path, monkeypatch, validate_crds):
+    FLOW_REGISTER.clear()
+    package_name = "selective_asset_package"
+    package_path = tmp_path / package_name
+    package_path.mkdir()
+    (package_path / "__init__.py").write_text("", encoding="utf-8")
+    (package_path / "flow.py").write_text(
+        'from tinyfan import Flow\nflow = Flow("selective-discovery")\n',
+        encoding="utf-8",
+    )
+    (package_path / "assets.py").write_text(
+        "import tinyfan as tf\n"
+        "from .flow import flow\n\n"
+        '@tf.asset(flow=flow, schedule="@daily")\n'
+        "def discovered_asset():\n"
+        '    return "hello"\n',
+        encoding="utf-8",
+    )
+    (package_path / "unrelated.py").write_text(
+        'raise RuntimeError("unrelated module was imported")\n',
+        encoding="utf-8",
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    try:
+        actual = codegen(location=package_name)
+
+        validate_crds(actual)
+        assert f"{package_name}.assets" in sys.modules
+        assert f"{package_name}.flow" in sys.modules
+        assert f"{package_name}.unrelated" not in sys.modules
+    finally:
+        for module_name in list(sys.modules):
+            if module_name == package_name or module_name.startswith(f"{package_name}."):
+                sys.modules.pop(module_name)
+        FLOW_REGISTER.clear()
+
+
+def test_codegen_import_all_supports_dynamic_asset_decorators(tmp_path, monkeypatch, validate_crds):
+    FLOW_REGISTER.clear()
+    package_name = "dynamic_asset_package"
+    package_path = tmp_path / package_name
+    package_path.mkdir()
+    (package_path / "__init__.py").write_text("", encoding="utf-8")
+    (package_path / "assets.py").write_text(
+        "from tinyfan import asset\n\n"
+        'scheduled_asset = asset(schedule="@daily")\n\n'
+        "@scheduled_asset\n"
+        "def dynamic_asset():\n"
+        '    return "hello"\n',
+        encoding="utf-8",
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    try:
+        codegen(location=package_name)
+        assert f"{package_name}.assets" not in sys.modules
+
+        actual = codegen(location=package_name, import_all=True)
+        validate_crds(actual)
+        assert f"{package_name}.assets" in sys.modules
+    finally:
+        for module_name in list(sys.modules):
+            if module_name == package_name or module_name.startswith(f"{package_name}."):
+                sys.modules.pop(module_name)
+        FLOW_REGISTER.clear()
+
+
 def test_rundata_data_interval(sample_flow):
     tree = AssetTree(sample_flow)
     node = list(tree.nodes.values())[0]

tinyfan-0.4.0/tests/test_gcs_pickle_store.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from tinyfan.stores.gcs_pickle import GcsPickleStore
+
+
+class FakeBlob:
+    def __init__(self, name: str, generation: int | None = None) -> None:
+        self.name = name
+        self.generation = generation
+        self.content_type: str | None = None
+        self.data: bytes | None = None
+
+    def upload_from_string(self, data: bytes, content_type: str) -> None:
+        self.data = data
+        self.content_type = content_type
+        self.generation = (self.generation or 0) + 1
+
+    def download_as_bytes(self) -> bytes:
+        assert self.data is not None
+        return self.data
+
+
+class FakeBucket:
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.blobs: dict[str, FakeBlob] = {}
+
+    def blob(self, name: str, generation: int | None = None) -> FakeBlob:
+        blob = self.blobs.setdefault(name, FakeBlob(name, generation))
+        blob.generation = generation or blob.generation
+        return blob
+
+
+class FakeClient:
+    def __init__(self) -> None:
+        self.buckets: dict[str, FakeBucket] = {}
+
+    def bucket(self, name: str) -> FakeBucket:
+        return self.buckets.setdefault(name, FakeBucket(name))
+
+
+def test_gcs_pickle_store_roundtrip() -> None:
+    client = FakeClient()
+    store = GcsPickleStore("test-bucket", prefix="runs", client=client)
+    value = {"hello": ["world", 1]}
+    rundata = {
+        "flow_name": "flow",
+        "asset_name": "asset",
+        "ds": "2026-05-31",
+        "data_interval_start": datetime(2026, 5, 31, 12, 30, tzinfo=UTC),
+    }
+
+    index = store.store(value, rundata)
+
+    assert index["bucket"] == "test-bucket"
+    assert index["name"].startswith("runs/flow/asset/2026-05-31T12-30-00-00-00/")
+    assert index["name"].endswith(".pickle")
+    assert index["generation"] == 1
+    assert client.bucket("test-bucket").blob(index["name"]).content_type == "application/octet-stream"
+    assert store.retrieve(index, source_rundata=rundata, target_rundata={}) == value
+
+
+def test_gcs_pickle_store_uses_metadata_blob_name() -> None:
+    store = GcsPickleStore("test-bucket", client=FakeClient())
+    rundata = {"metadata": {"blob_name": "/custom/path/value.pkl"}}
+
+    index = store.store("value", rundata)
+
+    assert index["name"] == "custom/path/value.pkl"

{tinyfan-0.2.9 → tinyfan-0.4.0}/uv.lock

@@ -2,6 +2,15 @@ version = 1
 revision = 3
 requires-python = ">=3.12"
 
+[[package]]
+name = "annotated-doc"
+version = "0.0.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" },
+]
+
 [[package]]
 name = "arrow"
 version = "1.3.0"
@@ -81,18 +90,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", size = 49446, upload-time = "2024-10-09T07:40:19.383Z" },
 ]
 
-[[package]]
-name = "click"
-version = "8.1.7"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "colorama", marker = "sys_platform == 'win32'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121, upload-time = "2023-08-17T17:29:11.868Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941, upload-time = "2023-08-17T17:29:10.08Z" },
-]
-
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -397,6 +394,27 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ee/07/44bd408781594c4d0a027666ef27fab1e441b109dc3b76b4f836f8fd04fe/jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c", size = 18482, upload-time = "2023-12-25T15:16:51.997Z" },
 ]
 
+[[package]]
+name = "markdown-it-py"
+version = "4.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mdurl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" },
+]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
+]
+
 [[package]]
 name = "mypy"
 version = "1.14.0"
@@ -591,6 +609,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537, upload-time = "2024-09-11T16:02:10.336Z" },
 ]
 
+[[package]]
+name = "pygments"
+version = "2.20.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
+]
+
 [[package]]
 name = "pytest"
 version = "8.3.3"
@@ -722,6 +749,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl", hash = "sha256:10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53", size = 13377, upload-time = "2018-07-29T17:23:45.313Z" },
 ]
 
+[[package]]
+name = "rich"
+version = "15.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markdown-it-py" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" },
+]
+
 [[package]]
 name = "rpds-py"
 version = "0.20.0"
@@ -793,6 +833,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3e/14/fd026bc74ded05e2351681545a5f626e78ef831f8edce064d61acd2e6ec7/ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93", size = 8679879, upload-time = "2024-10-04T13:40:25.797Z" },
 ]
 
+[[package]]
+name = "shellingham"
+version = "1.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -804,7 +853,7 @@ wheels = [
 
 [[package]]
 name = "tinyfan"
-version = "0.2.9"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },
@@ -854,7 +903,7 @@ viz = [
 
 [package.metadata]
 requires-dist = [
-    { name = "croniter", specifier = ">=6.0.0" },
+    { name = "croniter", specifier = ">=6" },
     { name = "google-cloud-bigquery", extras = ["pandas"], marker = "extra == 'bigquery'" },
     { name = "google-cloud-storage", marker = "extra == 'gcs'" },
     { name = "jsonpatch", marker = "extra == 'dev'", specifier = ">=1" },
@@ -872,7 +921,7 @@ requires-dist = [
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'all'" },
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'bigquery'" },
     { name = "tinyfan", extras = ["viz"], marker = "extra == 'all'" },
-    { name = "typer-slim", specifier = ">=0.20.0" },
+    { name = "typer-slim", specifier = ">=0.24.0" },
     { name = "types-croniter", marker = "extra == 'dev'" },
     { name = "types-jsonschema", marker = "extra == 'dev'" },
     { name = "types-networkx", marker = "extra == 'dev'" },
@@ -880,17 +929,31 @@ requires-dist = [
 ]
 provides-extras = ["all", "gcs", "viz", "bigquery", "pandas", "dev"]
 
+[[package]]
+name = "typer"
+version = "0.26.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "annotated-doc" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "rich" },
+    { name = "shellingham" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8e/d3/90c1ee19209cb59f6ad185883fd4ccfcf72f8f0bfd549d5a8b70474611d0/typer-0.26.4.tar.gz", hash = "sha256:25b128964de66c5ea36d5ac82adc579e5e113509b17469edf9f5a4a1864ff2a9", size = 201191, upload-time = "2026-05-30T17:05:04.213Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f0/6d/5a525c69df4a90892135e5d490b00e9e46402491f3416d4395fcb0d0201e/typer-0.26.4-py3-none-any.whl", hash = "sha256:11bfd7b43557137e373c2b10f6967a555f9678a61ed72c808968b011d95534d6", size = 122436, upload-time = "2026-05-30T17:05:05.812Z" },
+]
+
 [[package]]
 name = "typer-slim"
-version = "0.20.0"
+version = "0.24.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "click" },
-    { name = "typing-extensions" },
+    { name = "typer" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8e/45/81b94a52caed434b94da65729c03ad0fb7665fab0f7db9ee54c94e541403/typer_slim-0.20.0.tar.gz", hash = "sha256:9fc6607b3c6c20f5c33ea9590cbeb17848667c51feee27d9e314a579ab07d1a3", size = 106561, upload-time = "2025-10-20T17:03:46.642Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a7/a7/e6aecc4b4eb59598829a3b5076a93aff291b4fdaa2ded25efc4e1f4d219c/typer_slim-0.24.0.tar.gz", hash = "sha256:f0ed36127183f52ae6ced2ecb2521789995992c521a46083bfcdbb652d22ad34", size = 4776, upload-time = "2026-02-16T22:08:51.2Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5e/dd/5cbf31f402f1cc0ab087c94d4669cfa55bd1e818688b910631e131d74e75/typer_slim-0.20.0-py3-none-any.whl", hash = "sha256:f42a9b7571a12b97dddf364745d29f12221865acef7a2680065f9bb29c7dc89d", size = 47087, upload-time = "2025-10-20T17:03:44.546Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/24/5480c20380dfd18cf33d14784096dca45a24eae6102e91d49a718d3b6855/typer_slim-0.24.0-py3-none-any.whl", hash = "sha256:d5d7ee1ee2834d5020c7c616ed5e0d0f29b9a4b1dd283bdebae198ec09778d0e", size = 3394, upload-time = "2026-02-16T22:08:49.92Z" },
 ]
 
 [[package]]
@@ -916,14 +979,14 @@ wheels = [
 
 [[package]]
 name = "types-networkx"
-version = "3.5.0.20251106"
+version = "3.6.1.20260518"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/da/0d/0657390fb7e4fffb9e0a20d51b9b63451771c238ad7ca2eb31de12827a87/types_networkx-3.5.0.20251106.tar.gz", hash = "sha256:2f17a53a35cf6dbbb031039c19bf0f127ec1b37f3530c57a517a0dd67a42e343", size = 72307, upload-time = "2025-11-06T03:06:52.263Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/9f/e6fe2543aecbfe4d67fe86f280afae1e6d532c49755c5fb6c1f05f8af91e/types_networkx-3.6.1.20260518.tar.gz", hash = "sha256:af3ab153815b366c3695a0015cc75aece9b3eb0e5df56bb60ffee0b7cec8f630", size = 73977, upload-time = "2026-05-18T06:06:05.573Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/01/f768e7d3662841a5163f261510148a89564ffc4320568dfb98b79f8f6e68/types_networkx-3.5.0.20251106-py3-none-any.whl", hash = "sha256:674c53111d5ca6cf96eb6ccd3152d0d52c2720b8ddea8082c7b0116a21347a30", size = 160776, upload-time = "2025-11-06T03:06:50.89Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/4f/422a91e9fa008b804746fc2ce4da5bcb7eca4bfb45c31e8282fbb3664601/types_networkx-3.6.1.20260518-py3-none-any.whl", hash = "sha256:574304deea33828fb32cdca757af84f79b67bd80164aa1725c52a95d4a9347e6", size = 162454, upload-time = "2026-05-18T06:06:04.417Z" },
 ]
 
 [[package]]