tinyfan 0.3.0__tar.gz → 0.4.0__tar.gz

{tinyfan-0.3.0 → tinyfan-0.4.0}/.gitignore

@@ -8,3 +8,7 @@ wheels/
 
 # Virtual environments
 .venv
+
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/

{tinyfan-0.3.0 → tinyfan-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.3.0
+Version: 0.4.0
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -101,6 +101,18 @@ def greeting(world: str):
 tinyfan main.py | kubectl apply -f -
 ```
 
+For a package, pass its directory or import name. Tinyfan statically discovers
+modules containing `@asset`, so unrelated submodules are not executed during
+manifest generation. Modules imported by an asset module (for example, a shared
+`Flow` definition) are loaded normally.
+
+```shell
+tinyfan src/my_package
+```
+
+If a project registers assets dynamically or wraps `@asset` in another decorator,
+use `--import-all` to opt into the previous eager-import behavior.
+
 # Real World Example (still tiny though!)
 
 Comming soon

{tinyfan-0.3.0 → tinyfan-0.4.0}/README.md

@@ -50,6 +50,18 @@ def greeting(world: str):
 tinyfan main.py | kubectl apply -f -
 ```
 
+For a package, pass its directory or import name. Tinyfan statically discovers
+modules containing `@asset`, so unrelated submodules are not executed during
+manifest generation. Modules imported by an asset module (for example, a shared
+`Flow` definition) are loaded normally.
+
+```shell
+tinyfan src/my_package
+```
+
+If a project registers assets dynamically or wraps `@asset` in another decorator,
+use `--import-all` to opt into the previous eager-import behavior.
+
 # Real World Example (still tiny though!)
 
 Comming soon

{tinyfan-0.3.0 → tinyfan-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinyfan"
-version = "0.3.0"
+version = "0.4.0"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"

{tinyfan-0.3.0 → tinyfan-0.4.0}/src/tinyfan/cli.py

@@ -9,6 +9,10 @@ def tinyfan(
     location: str,
     embedded: bool | None = None,
     arg: Annotated[list[str] | None, typer.Option(metavar="KEY=VALUE")] = None,
+    import_all: Annotated[
+        bool,
+        typer.Option(help="Import every submodule instead of discovering modules with @asset statically."),
+    ] = False,
 ):
     """
     Generate argocd workflow resource as yaml from tinyfan definitions
@@ -16,9 +20,9 @@ def tinyfan(
     if arg is not None:
         CLI_CONFIG_STATE.cli_args.update({k: v for (k, v) in (val.split("=") for val in arg)})
     if location.endswith(".py"):
-        print(codegen(location=location, embedded=embedded or True))
+        print(codegen(location=location, embedded=embedded or True, import_all=import_all))
     else:
-        print(codegen(location=location, embedded=embedded or False))
+        print(codegen(location=location, embedded=embedded or False, import_all=import_all))
 
 
 def main():

{tinyfan-0.3.0 → tinyfan-0.4.0}/src/tinyfan/codegen.py

@@ -1,4 +1,5 @@
 from tinyfan.flow import Flow, Asset, FLOW_REGISTER, DEFAULT_IMAGE
+import ast
 import copy
 import importlib.util
 from typing import Self
@@ -9,7 +10,8 @@ import sys
 import re
 import json
 import os
-import pkgutil
+from pathlib import Path
+import tokenize
 import datetime
 import croniter
 from .argo_typing import ScriptTemplate, TemplateInputs, TemplateOutputs
@@ -28,7 +30,9 @@ INTERNAL_INPUT_PARAMETERS = {"rundata", "asset_name", "moduledata"}
 INTERNAL_OUTPUT_PARAMETERS = {"rundata"}
 
 
-def merge_template_io(template_io: dict, user_io: TemplateInputs | TemplateOutputs | None, reserved_parameter_names: set[str]) -> dict:
+def merge_template_io(
+    template_io: dict, user_io: TemplateInputs | TemplateOutputs | None, reserved_parameter_names: set[str]
+) -> dict:
     result = copy.deepcopy(template_io)
     if user_io is None:
         return result
@@ -75,7 +79,85 @@ def encode_schedule_to_k8sname(tz: str, cron: str) -> str:
     return f"{tz}|{cron}".lower().translate(str.maketrans("* ,-/_+|", "a-cds-p-", "@"))[:58]
 
 
-def import_all_submodules(location: str):
+def _defines_tinyfan_asset(source_path: Path) -> bool:
+    """Return whether a module has a statically recognizable Tinyfan asset."""
+    with tokenize.open(source_path) as source_file:
+        tree = ast.parse(source_file.read(), filename=str(source_path))
+
+    asset_names: set[str] = set()
+    tinyfan_namespaces: set[str] = set()
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for imported in node.names:
+                if imported.name == "tinyfan":
+                    tinyfan_namespaces.add(imported.asname or "tinyfan")
+                elif imported.name == "tinyfan.flow":
+                    tinyfan_namespaces.add(imported.asname or "tinyfan")
+        elif isinstance(node, ast.ImportFrom) and node.module in {"tinyfan", "tinyfan.flow"}:
+            for imported in node.names:
+                if imported.name == "asset":
+                    asset_names.add(imported.asname or imported.name)
+                elif imported.name == "*":
+                    asset_names.add("asset")
+
+    def is_asset_decorator(decorator: ast.expr) -> bool:
+        while isinstance(decorator, ast.Call):
+            decorator = decorator.func
+        if isinstance(decorator, ast.Name):
+            return decorator.id in asset_names
+
+        names: list[str] = []
+        while isinstance(decorator, ast.Attribute):
+            names.append(decorator.attr)
+            decorator = decorator.value
+        if isinstance(decorator, ast.Name):
+            names.append(decorator.id)
+        names.reverse()
+        return len(names) >= 2 and names[0] in tinyfan_namespaces and names[-1] == "asset"
+
+    class AssetVisitor(ast.NodeVisitor):
+        found = False
+
+        def _visit_definition(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
+            if any(is_asset_decorator(decorator) for decorator in node.decorator_list):
+                self.found = True
+
+        def visit_FunctionDef(self, node: ast.FunctionDef):
+            self._visit_definition(node)
+
+        def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+            self._visit_definition(node)
+
+        def visit_ClassDef(self, node: ast.ClassDef):
+            self._visit_definition(node)
+            if not self.found:
+                self.generic_visit(node)
+
+    visitor = AssetVisitor()
+    visitor.visit(tree)
+    return visitor.found
+
+
+def _package_source_modules(package) -> list[tuple[str, Path]]:
+    modules: list[tuple[str, Path]] = []
+    for package_path in package.__path__:
+        root = Path(package_path)
+        for source_path in root.rglob("*.py"):
+            relative_path = source_path.relative_to(root)
+            if source_path.name == "__init__.py":
+                module_parts = relative_path.parts[:-1]
+            else:
+                module_parts = relative_path.with_suffix("").parts
+            if not module_parts:
+                # Importing the root package has already executed its __init__.py.
+                continue
+            modules.append((".".join((package.__name__, *module_parts)), source_path))
+    return modules
+
+
+def import_asset_modules(location: str, import_all: bool = False):
+    """Import asset-bearing modules from a module name, package directory, or file."""
     try:
         package = importlib.import_module(location)
     except ModuleNotFoundError:
@@ -90,9 +172,16 @@ def import_all_submodules(location: str):
         package = importlib.util.module_from_spec(spec)
         sys.modules[spec.name] = package
         spec.loader.exec_module(package)
-    if package.__name__ != "__tinyfan_repo__":
-        for _, name, _ in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
-            importlib.import_module(name)
+    if package.__name__ == "__tinyfan_repo__" or not hasattr(package, "__path__"):
+        return
+    for module_name, source_path in _package_source_modules(package):
+        if import_all or _defines_tinyfan_asset(source_path):
+            importlib.import_module(module_name)
+
+
+def import_all_submodules(location: str):
+    """Import every submodule, preserving the eager discovery API used before 0.3.0."""
+    import_asset_modules(location, import_all=True)
 
 
 def assetname2template(name: str):
@@ -496,6 +585,7 @@ def codegen(
     embedded: bool = False,
     container: ScriptTemplate = {},
     serviceAccountName: str | None = None,
+    import_all: bool = False,
 ) -> str:
     """Generate argocd workflow resource as yaml from tinyfan definitions"""
     try:
@@ -503,7 +593,7 @@ def codegen(
         if flow:
             result = AssetTree(flow).compile(embedded, container, serviceAccountName)
         elif location:
-            import_all_submodules(location)
+            import_asset_modules(location, import_all=import_all)
             result = "\n---\n".join(
                 AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
             )

{tinyfan-0.3.0 → tinyfan-0.4.0}/tests/test_codegen.py

@@ -4,6 +4,7 @@ from tinyfan.codegen import codegen, AssetTree
 from jsonschema.exceptions import ValidationError
 from typing import Generator
 import os
+import sys
 import pytest
 import yaml
 
@@ -121,9 +122,7 @@ def test_codegen_asset_inputs_outputs(validate_crds):
         validate_crds(actual)
         manifest = next(yaml.load_all(actual, yaml.Loader))
         template = next(
-            template
-            for template in manifest["spec"]["workflowSpec"]["templates"]
-            if template["name"] == "asset1"
+            template for template in manifest["spec"]["workflowSpec"]["templates"] if template["name"] == "asset1"
         )
         input_params = {param["name"]: param for param in template["inputs"]["parameters"]}
         output_params = {param["name"]: param for param in template["outputs"]["parameters"]}
@@ -170,6 +169,74 @@ def test_codegen_directory(validate_crds):
     assert_code_is_running(actual)
 
 
+def test_codegen_directory_only_imports_asset_modules(tmp_path, monkeypatch, validate_crds):
+    FLOW_REGISTER.clear()
+    package_name = "selective_asset_package"
+    package_path = tmp_path / package_name
+    package_path.mkdir()
+    (package_path / "__init__.py").write_text("", encoding="utf-8")
+    (package_path / "flow.py").write_text(
+        'from tinyfan import Flow\nflow = Flow("selective-discovery")\n',
+        encoding="utf-8",
+    )
+    (package_path / "assets.py").write_text(
+        "import tinyfan as tf\n"
+        "from .flow import flow\n\n"
+        '@tf.asset(flow=flow, schedule="@daily")\n'
+        "def discovered_asset():\n"
+        '    return "hello"\n',
+        encoding="utf-8",
+    )
+    (package_path / "unrelated.py").write_text(
+        'raise RuntimeError("unrelated module was imported")\n',
+        encoding="utf-8",
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    try:
+        actual = codegen(location=package_name)
+
+        validate_crds(actual)
+        assert f"{package_name}.assets" in sys.modules
+        assert f"{package_name}.flow" in sys.modules
+        assert f"{package_name}.unrelated" not in sys.modules
+    finally:
+        for module_name in list(sys.modules):
+            if module_name == package_name or module_name.startswith(f"{package_name}."):
+                sys.modules.pop(module_name)
+        FLOW_REGISTER.clear()
+
+
+def test_codegen_import_all_supports_dynamic_asset_decorators(tmp_path, monkeypatch, validate_crds):
+    FLOW_REGISTER.clear()
+    package_name = "dynamic_asset_package"
+    package_path = tmp_path / package_name
+    package_path.mkdir()
+    (package_path / "__init__.py").write_text("", encoding="utf-8")
+    (package_path / "assets.py").write_text(
+        "from tinyfan import asset\n\n"
+        'scheduled_asset = asset(schedule="@daily")\n\n'
+        "@scheduled_asset\n"
+        "def dynamic_asset():\n"
+        '    return "hello"\n',
+        encoding="utf-8",
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    try:
+        codegen(location=package_name)
+        assert f"{package_name}.assets" not in sys.modules
+
+        actual = codegen(location=package_name, import_all=True)
+        validate_crds(actual)
+        assert f"{package_name}.assets" in sys.modules
+    finally:
+        for module_name in list(sys.modules):
+            if module_name == package_name or module_name.startswith(f"{package_name}."):
+                sys.modules.pop(module_name)
+        FLOW_REGISTER.clear()
+
+
 def test_rundata_data_interval(sample_flow):
     tree = AssetTree(sample_flow)
     node = list(tree.nodes.values())[0]

{tinyfan-0.3.0 → tinyfan-0.4.0}/uv.lock

@@ -853,7 +853,7 @@ wheels = [
 
 [[package]]
 name = "tinyfan"
-version = "0.3.0"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },
@@ -903,7 +903,7 @@ viz = [
 
 [package.metadata]
 requires-dist = [
-    { name = "croniter", specifier = ">=6.0.0" },
+    { name = "croniter", specifier = ">=6" },
     { name = "google-cloud-bigquery", extras = ["pandas"], marker = "extra == 'bigquery'" },
     { name = "google-cloud-storage", marker = "extra == 'gcs'" },
     { name = "jsonpatch", marker = "extra == 'dev'", specifier = ">=1" },